From 7658954140b98c1c3e56e9429c3613dccc3afe04 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 26 Sep 2026 02:26:33 +0200 Subject: [PATCH 01/34] fix(artifact): write and detect artifacts through wide paths on Windows (#1171) With persistence=true, indexing a repository under a CJK path on Windows made the worker exit nonzero and no .codebase-memory/graph.db.zst was written, while an ASCII junction to the same directory worked. Root cause: write_file_atomic() -- used for artifact.json, graph.db.zst and the import temp db -- opened its temp file with the raw CRT fopen() and published it with MoveFileExA(). Both go through the ANSI code page, so a repo path containing characters outside it cannot be opened. The same file had two more ANSI calls on the same flow: cbm_artifact_exists() used stat() on the repo path, and the import published the cache db with CRT rename(), which is ANSI and also fails when the destination exists. Fix: use the repo's UTF-8 helpers -- cbm_fopen() for the temp file, cbm_rename_replace() (MoveFileExW, write-through, replace-existing) for both publishes, and a cbm_fopen() probe for the non-empty existence check. rename_temp now reports errno on every platform (cbm_rename_replace translates the Win32 error). Test: artifact::artifact_roundtrip_non_ascii_paths exports from a CJK repo directory, checks cbm_artifact_exists(), and imports twice into a CJK cache directory (the second import replaces the existing db). Signed-off-by: Martin Vogel --- src/pipeline/artifact.c | 32 ++++++++++++++++--------------- tests/test_artifact.c | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/pipeline/artifact.c b/src/pipeline/artifact.c index 2dd368c28..9594c8a09 100644 --- a/src/pipeline/artifact.c +++ b/src/pipeline/artifact.c @@ -185,7 +185,10 @@ static int write_file_atomic(const char *path, const char *data, size_t len, return CBM_NOT_FOUND; } - FILE *fp = fopen(tmp, "wb"); + /* cbm_fopen / cbm_rename_replace: wide paths on Windows. The ANSI CRT fopen + * and MoveFileExA mangled a repo path outside the active code page (a CJK + * checkout), so every persistent export failed there (#1171). */ + FILE *fp = cbm_fopen(tmp, "wb"); if (!fp) { file_error_set(out_err, "open_temp", errno); return CBM_NOT_FOUND; @@ -207,22 +210,14 @@ static int write_file_atomic(const char *path, const char *data, size_t len, return CBM_NOT_FOUND; } -#ifdef _WIN32 - /* MoveFileEx replace approach suggested by @Ayush7Ranjan in #492. */ - if (!MoveFileExA(tmp, path, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { - DWORD saved_error = GetLastError(); - cbm_unlink(tmp); - file_error_set(out_err, "rename_temp", (int)saved_error); - return CBM_NOT_FOUND; - } -#else - if (rename(tmp, path) != 0) { + /* Replace-existing rename (MoveFileExW write-through on Windows, the + * approach suggested by @Ayush7Ranjan in #492); errno carries the cause. */ + if (cbm_rename_replace(tmp, path) != 0) { int saved_errno = errno; cbm_unlink(tmp); file_error_set(out_err, "rename_temp", saved_errno); return CBM_NOT_FOUND; } -#endif return 0; } @@ -1188,7 +1183,7 @@ int cbm_artifact_import(const char *repo_path, const char *cache_db_path) { * stale WAL next to the cache path would be replayed on top of the * imported file at the next open (#897). */ cbm_remove_db_sidecars(cache_db_path); - if (rename(tmp_path, cache_db_path) != 0) { + if (cbm_rename_replace(tmp_path, cache_db_path) != 0) { cbm_log_error("artifact.import", "err", "rename_to_cache"); cbm_unlink(tmp_path); return CBM_NOT_FOUND; @@ -1218,8 +1213,15 @@ bool cbm_artifact_exists(const char *repo_path) { char zst_path[CBM_SZ_4K]; artifact_path(zst_path, sizeof(zst_path), repo_path, CBM_ARTIFACT_FILENAME); - struct stat st; - if (stat(zst_path, &st) != 0 || st.st_size == 0) { + /* Present and non-empty. cbm_fopen, not stat(): the ANSI CRT stat misses a + * repo path outside the active Windows code page (#1171). */ + FILE *fp = cbm_fopen(zst_path, "rb"); + if (!fp) { + return false; + } + int first = fgetc(fp); + (void)fclose(fp); + if (first == EOF) { return false; } diff --git a/tests/test_artifact.c b/tests/test_artifact.c index a7d96cdd9..6206463e3 100644 --- a/tests/test_artifact.c +++ b/tests/test_artifact.c @@ -334,6 +334,47 @@ TEST(artifact_gitattributes_created) { PASS(); } +/* #1171: a repository (and a cache directory) under a non-ASCII path must + * export, report the artifact present, and import — also onto an existing + * cache db. On Windows the atomic writer used the ANSI CRT fopen + MoveFileExA + * and the existence check the ANSI stat, so CJK characters outside the active + * code page made every step fail. The bytes are escaped UTF-8 for a CJK repo + * directory name and a CJK cache directory name (kept ASCII in source). */ +#define ART_CJK_REPO "\xe6\x8c\x81\xe4\xb9\x85\xe5\x8c\x96\xe5\xa4\x8d\xe7\x8e\xb0" +#define ART_CJK_CACHE "\xe5\xaf\xbc\xe5\x85\xa5" +TEST(artifact_roundtrip_non_ascii_paths) { + setup_artifact_test(); + create_test_db(g_db); + + char repo[1024]; + snprintf(repo, sizeof(repo), "%s/cbm-" ART_CJK_REPO, g_tmpdir); + ASSERT_TRUE(cbm_mkdir_p(repo, 0755)); + + ASSERT_EQ(cbm_artifact_export(g_db, repo, "test-proj", CBM_ARTIFACT_FAST), 0); + char zst[1024]; + snprintf(zst, sizeof(zst), "%s/.codebase-memory/graph.db.zst", repo); + ASSERT_TRUE(cbm_file_exists(zst)); + ASSERT_TRUE(cbm_artifact_exists(repo)); + + char cache_dir[1024]; + snprintf(cache_dir, sizeof(cache_dir), "%s/" ART_CJK_CACHE, g_tmpdir); + ASSERT_TRUE(cbm_mkdir_p(cache_dir, 0755)); + char import_db[1024]; + snprintf(import_db, sizeof(import_db), "%s/imported.db", cache_dir); + ASSERT_EQ(cbm_artifact_import(repo, import_db), 0); + /* Re-import replaces the existing cache db (the second-session path). */ + ASSERT_EQ(cbm_artifact_import(repo, import_db), 0); + + cbm_store_t *s = cbm_store_open_path(import_db); + ASSERT_NOT_NULL(s); + ASSERT_EQ(cbm_store_count_nodes(s, "test-proj"), 2); + ASSERT_EQ(cbm_store_count_edges(s, "test-proj"), 1); + cbm_store_close(s); + + cleanup_dir(g_tmpdir); + PASS(); +} + TEST(artifact_export_rename_failure_logs_specific_error) { setup_artifact_test(); create_test_db(g_db); @@ -1102,6 +1143,7 @@ SUITE(artifact) { RUN_TEST(artifact_schema_version_mismatch); RUN_TEST(artifact_import_missing); RUN_TEST(artifact_gitattributes_created); + RUN_TEST(artifact_roundtrip_non_ascii_paths); RUN_TEST(artifact_export_rename_failure_logs_specific_error); RUN_TEST(pipeline_persistence_export_failure_returns_error); RUN_TEST(artifact_import_rejects_size_mismatch); From 431778c0ae420a3eedb9a795d96f77959d569029 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 17:33:05 +0200 Subject: [PATCH 02/34] fix(extract): a decorator with arguments but no path is not a route Distilled from #1245 by Andrew Hundt (a slice of d15071c9), with a different rule than upstream chose. Python's unittest.mock is used as a decorator: @patch("subprocess.run") def test_runs(self, run): ... decorator_method_name maps a bare `patch` to the HTTP method PATCH, and try_route_from_decorator_call accepts a receiver-less decorator call. When extract_route_path_from_args then finds nothing path-shaped, the function falls back to "/". So every mock-decorated test function became a Route handler, PATCH "/", and every one of those was wrong. Upstream fixed it with a has_receiver heuristic. That still admits @mock.patch("os.getcwd") (it has a receiver) and would reject the bare framework decorators Litestar and BlackSheep use, @get("/x"). The rule here is about the arguments, not the callee: a decorator call that HAS arguments but none of them path-shaped is not a route. The "/" default survives only for a zero-argument call such as @app.route(), which is the one case where "/" is what the framework means. No test on main relied on the default for a call with non-path arguments; checked before changing it. RED before the fix: extract_python_mock_patch_is_not_route FAIL tests/test_extraction.c:4023: mocked->route_path is not NULL extract_python_bare_decorator_route_rules FAIL tests/test_extraction.c:4063: mocked->route_path is not NULL The second test also pins the two shapes that must keep working: bare @get("/health") still yields a GET route, and zero-argument @app.route() still yields "/" ANY -- those assertions passed before the fix and after. GREEN after: extraction 351 passed; extraction pipeline edge_types_probe route_canon cross_repo infrascan lang_contract 754 passed, with the CALLS-breadth contract at 54 languages / 0 failures. The single remaining red in both runs is extract_spill_round_trip_keeps_every_field refusing to spill onto a disk below its 10 GB floor -- this host, not this change. Co-authored-by: Andrew Hundt Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 9 ++++- tests/test_extraction.c | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 49543e59a..3fb279a11 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -1605,7 +1605,11 @@ static bool try_drf_action_decorator(CBMArena *a, TSNode dchild, const char *sou } // Try to extract a route from a single decorator call node. -// Returns true if a route method was found (even with fallback path "/"). +// Returns true if a route method was found. The fallback path "/" applies only +// to a zero-argument call (`@app.route()`): a call that HAS arguments but none +// path-shaped is not a route -- unittest.mock's `@patch("subprocess.run")` +// shares its name with the HTTP verb and used to mint a PATCH "/" handler for +// every mocked test function (distilled from PR #1245). static bool try_route_from_decorator_call(CBMArena *a, TSNode dchild, const char *source, const char **out_path, const char **out_method) { TSNode fn = ts_node_child_by_field_name(dchild, TS_FIELD("function")); @@ -1630,6 +1634,9 @@ static bool try_route_from_decorator_call(CBMArena *a, TSNode dchild, const char *out_method = method; return true; } + if (ts_node_named_child_count(args) > 0) { + return false; + } } *out_path = "/"; *out_method = method; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 50a0db788..6ded6d282 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4303,6 +4303,77 @@ TEST(extract_java_method_annotations_issue382) { PASS(); } +/* Distilled from PR #1245 (Andrew Hundt): unittest.mock's `@patch("x")` is a + * bare decorator whose name collides with the HTTP verb, and its only argument + * is a dotted target, not a path. The "/" fallback then minted a Route handler + * PATCH "/" for every mocked test function. Rule: a decorator call WITH + * arguments but no path-shaped one is not a route. */ +TEST(extract_python_mock_patch_is_not_route) { + CBMFileResult *r = extract("from unittest.mock import patch\n\n" + "@patch(\"subprocess.run\")\n" + "def test_cmd(mock_run):\n" + " pass\n\n" + "@app.patch(\"/items/{id}\")\n" + "def update_item():\n" + " pass\n", + CBM_LANG_PYTHON, "t", "test_routes.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *mocked = find_def_by_name(r, "test_cmd"); + ASSERT_NOT_NULL(mocked); + ASSERT_NULL(mocked->route_path); + ASSERT_NULL(mocked->route_method); + + const CBMDefinition *route = find_def_by_name(r, "update_item"); + ASSERT_NOT_NULL(route); + ASSERT_STR_EQ(route->route_path, "/items/{id}"); + ASSERT_STR_EQ(route->route_method, "PATCH"); + + cbm_free_result(r); + PASS(); +} + +/* Companion to the mock-patch case: the narrowing must not cost real routes. + * A receiver-less framework decorator with a path-shaped argument (Litestar / + * BlackSheep `@get("/x")`) stays a route, `@mock.patch("x")` (receiver, but + * no path) is not one, and a zero-argument `@app.route()` keeps the "/" + * default main emits today. */ +TEST(extract_python_bare_decorator_route_rules) { + CBMFileResult *r = extract("from litestar import get\n" + "from unittest import mock\n\n" + "@get(\"/health\")\n" + "def health():\n" + " pass\n\n" + "@mock.patch(\"os.getcwd\")\n" + "def test_cwd(mock_cwd):\n" + " pass\n\n" + "@app.route()\n" + "def index():\n" + " pass\n", + CBM_LANG_PYTHON, "t", "routes.py"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *health = find_def_by_name(r, "health"); + ASSERT_NOT_NULL(health); + ASSERT_STR_EQ(health->route_path, "/health"); + ASSERT_STR_EQ(health->route_method, "GET"); + + const CBMDefinition *mocked = find_def_by_name(r, "test_cwd"); + ASSERT_NOT_NULL(mocked); + ASSERT_NULL(mocked->route_path); + ASSERT_NULL(mocked->route_method); + + const CBMDefinition *index = find_def_by_name(r, "index"); + ASSERT_NOT_NULL(index); + ASSERT_STR_EQ(index->route_path, "/"); + ASSERT_STR_EQ(index->route_method, "ANY"); + + cbm_free_result(r); + PASS(); +} + /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { @@ -8813,6 +8884,8 @@ SUITE(extraction) { RUN_TEST(js_index_module_qn_not_collide_with_folder); RUN_TEST(python_regular_module_qn_unchanged); RUN_TEST(extract_java_method_annotations_issue382); + RUN_TEST(extract_python_mock_patch_is_not_route); + RUN_TEST(extract_python_bare_decorator_route_rules); RUN_TEST(arkts_component_struct); RUN_TEST(arkts_exported_struct_decorators); RUN_TEST(arkts_member_decorators); From e4f9735dea30dcc799ca5153a666ecdf9a00ca00 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 25 Sep 2026 20:08:36 +0200 Subject: [PATCH 03/34] feat(cross-repo): report run status and skipped legacy stores (#2133) Two follow-ups to the #2133 wildcard fix, both about what cross-repo intelligence tells the caller. 1. "Never run" vs "ran, no links". An empty CROSS_* set used to mean either one, and no output surface could tell them apart. Every run now records a marker in the SOURCE store's store_meta (key cross_repo_last_run:, same per-project key pattern as coverage_shadow_fp:) once the previous CROSS_* generation has been cleaned: last_run_at, outcome (complete|cancelled|failed), targets, projects_scanned, total_cross_edges, skipped_projects. index_status and the UI layout (next to linked_projects) now carry a `cross_repo` object: {"status":"never_run"} or {"status":"ran", ...}. The marker lives beside the edges it describes, so a reindex that rebuilds the store drops both together. No new database files and no new freshness key. New store API cbm_store_meta_get/put. put seeds a missing store_meta through the generation path, because creating the table without db_uid/mutation_gen would make cbm_store_generation() report malformed metadata. 2. ["*"] skips a pre-#768 store (since #2343) but only logged it. The result now names each one, and the MCP response lists them: skipped_projects: [{name, reason:"pre_768_schema", hint:"reindex this project"}] (only emitted when non-empty). The cross_repo.project_unusable log line is unchanged. cr_project_exists() is split into cr_project_state() (usable / missing / legacy) so that the enumerator can tell a legacy store apart from a foreign .db file. New allocations go through mem_core. Refs #2133 Signed-off-by: Martin Vogel --- README.md | 2 + src/mcp/mcp.c | 16 ++++ src/pipeline/pass_cross_repo.c | 169 ++++++++++++++++++++++++++++++--- src/pipeline/pass_cross_repo.h | 24 +++++ src/store/store.c | 71 ++++++++++++++ src/store/store.h | 11 +++ src/ui/http_server.c | 4 + tests/test_cross_repo.c | 93 ++++++++++++++++++ tests/test_mcp.c | 97 +++++++++++++++++++ 9 files changed, 476 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7564427ea..2500d6c51 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,8 @@ The install script placed beside the binary is **reported, not deleted** — uni - **`CROSS_*` edges** link nodes across multiple repos indexed under the same store - **Multi-galaxy 3D UI layout** for cross-repo architecture visualization - **Cross-repo architecture summary** combining services, routes, and dependencies across the indexed fleet +- **Run status**: `index_status` (and the UI layout next to `linked_projects`) carries a `cross_repo` object, `{"status":"never_run"}` until `index_repository(mode="cross-repo-intelligence")` has run with that project as the source, then `"ran"` with `last_run_at`, `outcome`, `targets`, `projects_scanned`, `total_cross_edges` and `skipped_projects`. That is how you tell "never linked" apart from "linked, nothing matched". The status is stored in the project's own index next to its `CROSS_*` edges, so a reindex that rebuilds the index drops both at once and the status goes back to `never_run`. +- **Skipped stores**: `target_projects: ["*"]` skips an index built before the #768 schema change rather than failing the whole run. The response lists each one as `skipped_projects: [{"name", "reason": "pre_768_schema", "hint": "reindex this project"}]`. ### Edge types (selected) - `CALLS` — a callable is invoked at the source site diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 2ed29799b..e67595882 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -6906,6 +6906,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { } add_coverage_report(doc, root, store, project, have_proj_info ? proj_info.indexed_at : NULL, coverage_samples); + (void)cbm_cross_repo_add_status_json(doc, root, store, project); safe_str_free(&proj_info.name); safe_str_free(&proj_info.indexed_at); safe_str_free(&proj_info.root_path); @@ -10054,6 +10055,7 @@ static char *handle_cross_repo_mode(cbm_mcp_server_t *srv, const char *repo_path yyjson_doc_free(jdoc); if (result.no_targets || result.failed) { + cbm_cross_repo_result_free(&result); free(project); return cbm_mcp_text_result( result.no_targets @@ -10091,9 +10093,23 @@ static char *handle_cross_repo_mode(cbm_mcp_server_t *srv, const char *repo_path yyjson_mut_obj_add_int(doc, root, "cross_trpc_calls", result.trpc_edges); yyjson_mut_obj_add_int(doc, root, "total_cross_edges", total); yyjson_mut_obj_add_real(doc, root, "elapsed_ms", result.elapsed_ms); + /* ["*"] skips stores it cannot link (#2133); name them so a partial fleet + * is visible instead of silently smaller. Only emitted when non-empty. */ + if (result.skipped_count > 0) { + yyjson_mut_val *skipped = yyjson_mut_arr(doc); + for (int i = 0; i < result.skipped_count; i++) { + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, item, "name", result.skipped_projects[i].project); + yyjson_mut_obj_add_str(doc, item, "reason", result.skipped_projects[i].reason); + yyjson_mut_obj_add_str(doc, item, "hint", "reindex this project"); + yyjson_mut_arr_append(skipped, item); + } + yyjson_mut_obj_add_val(doc, root, "skipped_projects", skipped); + } char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); + cbm_cross_repo_result_free(&result); free(project); char *out = cbm_mcp_text_result(json, result.cancelled); free(json); diff --git a/src/pipeline/pass_cross_repo.c b/src/pipeline/pass_cross_repo.c index b86d8cee4..f645db349 100644 --- a/src/pipeline/pass_cross_repo.c +++ b/src/pipeline/pass_cross_repo.c @@ -13,12 +13,14 @@ #include "pipeline/pipeline_internal.h" // cbm_route_canon_path #include "foundation/constants.h" #include "foundation/log.h" +#include "foundation/mem_core.h" #include "foundation/platform.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/str_util.h" #include +#include #include #include #include @@ -153,27 +155,38 @@ static bool cr_store_has_exact_project(cbm_store_t *store, const char *project) return matches; } -/* True when `project` is a usable cross-repo input: its store holds exactly - * that project AND carries the schema the read-write open in +typedef enum { + CR_PROJECT_USABLE = 0, + CR_PROJECT_MISSING, /* no store, or the store is not exactly this project */ + CR_PROJECT_LEGACY, /* exact project, but a pre-#768 schema (reindex required) */ +} cr_project_state_t; + +/* Classify `project` as a cross-repo input: usable when its store holds + * exactly that project AND carries the schema the read-write open in * cr_open_existing_project requires. Checking only the first let a pre-#768 * store (still readable, so list_projects shows it) pass validation and the * ["*"] enumeration, then abort the whole run at its write open — after the * source's previous CROSS_* generation had already been deleted. The probe is * read-only, so the legacy store is left untouched for its reindex. (#2133) */ -static bool cr_project_exists(const char *project) { +static cr_project_state_t cr_project_state(const char *project) { char path[CR_PATH_BUF]; if (!cr_db_path(project, path, sizeof(path))) { - return false; + return CR_PROJECT_MISSING; } cbm_store_t *store = cbm_store_open_path_query(path); - bool exists = cr_store_has_exact_project(store, project); - if (exists && !cbm_store_edges_schema_current(store)) { + cr_project_state_t state = + cr_store_has_exact_project(store, project) ? CR_PROJECT_USABLE : CR_PROJECT_MISSING; + if (state == CR_PROJECT_USABLE && !cbm_store_edges_schema_current(store)) { cbm_log_warn("cross_repo.project_unusable", "project", project, "reason", "pre_768_schema_reindex_required"); - exists = false; + state = CR_PROJECT_LEGACY; } cbm_store_close(store); - return exists; + return state; +} + +static bool cr_project_exists(const char *project) { + return cr_project_state(project) == CR_PROJECT_USABLE; } static cbm_store_t *cr_open_existing_project(const char *project) { @@ -1139,8 +1152,33 @@ static cr_match_result_t match_typed_routes(cbm_store_t *src_store, const char * static void free_project_list(char **projects, int count); +/* Remember a store the ["*"] enumeration skipped, so the caller can name it + * instead of the run silently covering fewer projects than exist. */ +static bool cr_record_skip(cbm_cross_repo_result_t *result, const char *project, + const char *reason) { + if (!result) { + return true; + } + cbm_cross_repo_skip_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, result->skipped_projects, + ((size_t)result->skipped_count + 1U) * sizeof(*result->skipped_projects)); + if (!grown) { + return false; + } + result->skipped_projects = grown; + char *name = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, project); + if (!name) { + return false; + } + grown[result->skipped_count].project = name; + grown[result->skipped_count].reason = reason; + result->skipped_count++; + return true; +} + /* When target_projects = ["*"], scan the cache directory for all .db files. */ -static int collect_all_projects(char ***out, cr_run_context_t *ctx) { +static int collect_all_projects(char ***out, cr_run_context_t *ctx, + cbm_cross_repo_result_t *result) { *out = NULL; const char *dir = cr_cache_dir(); cbm_dir_t *d = cbm_opendir(dir); @@ -1188,7 +1226,15 @@ static int collect_all_projects(char ***out, cr_run_context_t *ctx) { } memcpy(project, ent->name, project_length); project[project_length] = '\0'; - if (!cbm_validate_project_name(project) || !cr_project_exists(project)) { + if (!cbm_validate_project_name(project)) { + continue; + } + cr_project_state_t state = cr_project_state(project); + if (state == CR_PROJECT_LEGACY && !cr_record_skip(result, project, "pre_768_schema")) { + failed = true; + break; + } + if (state != CR_PROJECT_USABLE) { continue; } if (count >= cap) { @@ -1273,6 +1319,106 @@ static int collect_named_projects(const char **targets, int target_count, char * return unique_count; } +/* ── Last-run marker ─────────────────────────────────────────────── */ + +static bool cr_status_key(const char *project, char *buf, size_t bufsz) { + int written = snprintf(buf, bufsz, "cross_repo_last_run:%s", project); + return written > 0 && (size_t)written < bufsz; +} + +/* Record this run in the SOURCE store's store_meta, next to the CROSS_* + * edges it just rewrote: a reindex that rebuilds the store drops both + * together, so the marker can never outlive the links it describes. Written + * once the source's previous CROSS_* generation was cleaned, whatever the + * outcome, because from then on the old summary no longer matches the graph. + * Project names are validated ([A-Za-z0-9._-]) or "*", so no JSON escaping is + * needed. Best-effort: a marker failure is logged, never fails the run. */ +static void cr_record_last_run(cbm_store_t *src_store, const char *project, const char **targets, + int target_count, const cbm_cross_repo_result_t *result) { + char key[CR_QN_BUF]; + if (!cr_status_key(project, key, sizeof(key))) { + return; + } + size_t cap = CR_PROPS_BUF; + for (int i = 0; i < target_count; i++) { + cap += (targets[i] ? strlen(targets[i]) : 0) + CR_COL_4; + } + char *json = cbm_alloc(CBM_MEM_CLASS_OTHER, cap); + if (!json) { + return; + } + char when[CBM_SZ_32]; + time_t now = time(NULL); + struct tm tm_now; + if (!cbm_gmtime_r(&now, &tm_now) || + strftime(when, sizeof(when), "%Y-%m-%dT%H:%M:%SZ", &tm_now) == 0) { + when[0] = '\0'; + } + const char *outcome = result->failed ? "failed" : result->cancelled ? "cancelled" : "complete"; + size_t len = (size_t)snprintf( + json, cap, "{\"last_run_at\":\"%s\",\"outcome\":\"%s\",\"targets\":[", when, outcome); + for (int i = 0; i < target_count && len < cap; i++) { + len += (size_t)snprintf(json + len, cap - len, "%s\"%s\"", i ? "," : "", + targets[i] ? targets[i] : ""); + } + int total = result->http_edges + result->async_edges + result->channel_edges + + result->grpc_edges + result->graphql_edges + result->trpc_edges; + if (len < cap) { + len += (size_t)snprintf(json + len, cap - len, + "],\"projects_scanned\":%d,\"total_cross_edges\":%d," + "\"skipped_projects\":%d}", + result->projects_scanned, total, result->skipped_count); + } + if (len >= cap || cbm_store_meta_put(src_store, key, json) != CBM_STORE_OK) { + cbm_log_warn("cross_repo.status_unrecorded", "project", project); + } + cbm_free(CBM_MEM_CLASS_OTHER, json); +} + +bool cbm_cross_repo_add_status_json(struct yyjson_mut_doc *doc, struct yyjson_mut_val *parent, + cbm_store_t *store, const char *project) { + char key[CR_QN_BUF]; + if (!doc || !parent || !store || !project || !cr_status_key(project, key, sizeof(key))) { + return false; + } + char *raw = NULL; + int rc = cbm_store_meta_get(store, key, &raw); + yyjson_doc *recorded = rc == CBM_STORE_OK ? yyjson_read(raw, strlen(raw), 0) : NULL; + cbm_free(CBM_MEM_CLASS_STORE, raw); + yyjson_val *recorded_root = recorded ? yyjson_doc_get_root(recorded) : NULL; + yyjson_mut_val *status = NULL; + if (yyjson_is_obj(recorded_root)) { + status = yyjson_val_mut_copy(doc, recorded_root); + } + if (!status) { + status = yyjson_mut_obj(doc); + } + yyjson_doc_free(recorded); + if (!status) { + return false; + } + /* "never_run" is the honest reading of an absent marker: no run with this + * project as the source has completed its cleanup since the store was + * (re)built. A target-side run still leaves its CROSS_* edges here. */ + const char *state = rc == CBM_STORE_NOT_FOUND ? "never_run" + : yyjson_mut_obj_size(status) > 0 ? "ran" + : "unavailable"; + yyjson_mut_obj_add_str(doc, status, "status", state); + return yyjson_mut_obj_add_val(doc, parent, "cross_repo", status); +} + +void cbm_cross_repo_result_free(cbm_cross_repo_result_t *result) { + if (!result) { + return; + } + for (int i = 0; i < result->skipped_count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, result->skipped_projects[i].project); + } + cbm_free(CBM_MEM_CLASS_OTHER, result->skipped_projects); + result->skipped_projects = NULL; + result->skipped_count = 0; +} + static cr_run_status_t add_match_count(int *total, cr_match_result_t matched) { *total += matched.count; return matched.status; @@ -1304,7 +1450,7 @@ cbm_cross_repo_result_t cbm_cross_repo_match_cancellable(const char *project, int resolved_count = 0; if (target_count == SKIP_ONE && target_projects[0] && strcmp(target_projects[0], "*") == 0) { - resolved_count = collect_all_projects(&resolved, &run); + resolved_count = collect_all_projects(&resolved, &run, &result); } else { resolved_count = collect_named_projects(target_projects, target_count, &resolved, &run); } @@ -1442,6 +1588,7 @@ cbm_cross_repo_result_t cbm_cross_repo_match_cancellable(const char *project, result.projects_scanned++; } + cr_record_last_run(src_store, project, target_projects, target_count, &result); cbm_store_close(src_store); free_project_list(resolved, resolved_count); diff --git a/src/pipeline/pass_cross_repo.h b/src/pipeline/pass_cross_repo.h index 898283c3e..85c832b2d 100644 --- a/src/pipeline/pass_cross_repo.h +++ b/src/pipeline/pass_cross_repo.h @@ -9,6 +9,12 @@ #include +/* A store the ["*"] enumeration skipped instead of linking. */ +typedef struct { + char *project; /* heap-owned */ + const char *reason; /* static, e.g. "pre_768_schema" (reindex required) */ +} cbm_cross_repo_skip_t; + /* Result of a cross-repo matching run. */ typedef struct { int http_edges; /* CROSS_HTTP_CALLS edges created */ @@ -23,8 +29,26 @@ typedef struct { bool no_targets; /* failed: nothing but the source project resolved (#1133) */ bool cancelled; /* stopped at a bounded cancellation checkpoint */ bool partial_results; /* committed writes before cancellation were retained */ + cbm_cross_repo_skip_t *skipped_projects; /* ["*"] only; free with ..._result_free */ + int skipped_count; } cbm_cross_repo_result_t; +/* Release the heap-owned parts of a result (safe on a zeroed result). */ +void cbm_cross_repo_result_free(cbm_cross_repo_result_t *result); + +struct yyjson_mut_doc; +struct yyjson_mut_val; + +/* Add a "cross_repo" object to `parent` describing the last cross-repo run + * with `project` as the source, as recorded in that project's store: + * {"status":"never_run"} -- no run recorded since the store was (re)built + * {"status":"ran","last_run_at","outcome","targets","projects_scanned", + * "total_cross_edges","skipped_projects"} + * Read-only (works on a query store). Returns false only on invalid input or + * allocation failure. */ +bool cbm_cross_repo_add_status_json(struct yyjson_mut_doc *doc, struct yyjson_mut_val *parent, + cbm_store_t *store, const char *project); + /* Run cross-repo matching for `project` against `target_projects`. * If target_count == 1 and target_projects[0] == "*", matches against all * indexed projects. Writes CROSS_* edges bidirectionally into both the diff --git a/src/store/store.c b/src/store/store.c index d66ef0ffb..430efd8a3 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -2456,6 +2456,77 @@ int cbm_store_generation(cbm_store_t *s, char *buf, size_t bufsz) { return CBM_STORE_OK; } +int cbm_store_meta_get(cbm_store_t *s, const char *key, char **out) { + if (!out) { + return CBM_STORE_ERR; + } + *out = NULL; + if (!s || !s->db || !key) { + return CBM_STORE_ERR; + } + sqlite3_stmt *get = NULL; + /* A store that never advanced its generation has no store_meta table: + * that is "no value recorded", not an error. */ + if (sqlite3_prepare_v2(s->db, "SELECT v FROM store_meta WHERE k = ?1;", CBM_NOT_FOUND, &get, + NULL) != SQLITE_OK) { + sqlite3_finalize(get); + return CBM_STORE_NOT_FOUND; + } + bind_text(get, SKIP_ONE, key); + int rc = sqlite3_step(get); + if (rc == SQLITE_ROW) { + const char *value = (const char *)sqlite3_column_text(get, 0); + *out = cbm_mem_strdup(CBM_MEM_CLASS_STORE, value ? value : ""); + sqlite3_finalize(get); + return *out ? CBM_STORE_OK : CBM_STORE_ERR; + } + sqlite3_finalize(get); + return rc == SQLITE_DONE ? CBM_STORE_NOT_FOUND : CBM_STORE_ERR; +} + +int cbm_store_meta_put(cbm_store_t *s, const char *key, const char *value) { + static const char savepoint[] = "SAVEPOINT cbm_meta_put;"; + static const char rollback[] = "ROLLBACK TO cbm_meta_put;"; + static const char release[] = "RELEASE cbm_meta_put;"; + if (!s || !s->db || !key || !value) { + return CBM_STORE_ERR; + } + if (exec_sql(s, savepoint) != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + /* store_meta is only ever created by the generation seed, which writes + * db_uid + mutation_gen with it. Creating the table here without them + * would make cbm_store_generation() report malformed metadata, so an + * absent table is seeded through that one path. */ + bool absent = false; + char uid[17] = {0}; + char counter[21] = {0}; + uint64_t counter_value = 0; + if (generation_metadata_read(s, &absent, uid, counter, &counter_value) != CBM_STORE_OK || + (absent && cbm_store_generation_advance(s) != CBM_STORE_OK)) { + return rollback_savepoint_preserving_error(s, rollback, release); + } + sqlite3_stmt *set = NULL; + if (sqlite3_prepare_v2(s->db, "INSERT OR REPLACE INTO store_meta (k, v) VALUES (?1, ?2);", + CBM_NOT_FOUND, &set, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "store_meta put prepare"); + sqlite3_finalize(set); + return rollback_savepoint_preserving_error(s, rollback, release); + } + bind_text(set, SKIP_ONE, key); + bind_text(set, ST_COL_2, value); + int rc = sqlite3_step(set); + sqlite3_finalize(set); + if (rc != SQLITE_DONE) { + store_set_error_sqlite(s, "store_meta put"); + return rollback_savepoint_preserving_error(s, rollback, release); + } + if (exec_sql(s, release) != CBM_STORE_OK) { + return rollback_savepoint_preserving_error(s, rollback, release); + } + return CBM_STORE_OK; +} + int cbm_store_get_project(cbm_store_t *s, const char *name, cbm_project_t *out) { sqlite3_stmt *stmt = prepare_cached(s, &s->stmt_get_project, diff --git a/src/store/store.h b/src/store/store.h index 06ea1fbb7..ca4f1eb58 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -462,6 +462,17 @@ int cbm_store_generation_advance(cbm_store_t *s); * an error. */ int cbm_store_generation(cbm_store_t *s, char *buf, size_t bufsz); +/* Per-store feature metadata in store_meta (key/value text). Keys are + * namespaced by the owning feature, e.g. "cross_repo_last_run:". + * get: CBM_STORE_OK with *out owned by the caller (release with + * cbm_free(CBM_MEM_CLASS_STORE, *out)), or + * CBM_STORE_NOT_FOUND when the key -- or store_meta itself -- is absent. + * Read-only: safe on a store opened for query. + * put: upserts the key; seeds store_meta through the generation path when a + * legacy store lacks it. Atomic (savepoint); needs a read-write store. */ +int cbm_store_meta_get(cbm_store_t *s, const char *key, char **out); +int cbm_store_meta_put(cbm_store_t *s, const char *key, const char *value); + /* Seal a fully-written staging database before atomic publication. * Raises synchronous to FULL, requires an exclusive TRUNCATE checkpoint to * complete, then leaves the database in verified DELETE journal mode so the diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 66f1affb8..ae0dd2754 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -19,6 +19,7 @@ #include "ui/layout3d.h" #include "mcp/mcp.h" #include "store/store.h" +#include "pipeline/pass_cross_repo.h" #include "watcher/watcher.h" #include "cli/cli.h" #include "git/git_context.h" @@ -1729,6 +1730,9 @@ static void handle_layout(cbm_http_conn_t *c, const cbm_http_req_t *req) { free(linked[li]); } + /* Next to linked_projects: an empty list alone cannot tell "never linked" + * from "linked, nothing matched". */ + (void)cbm_cross_repo_add_status_json(mdoc, mroot, store, project); cbm_store_close(store); yyjson_mut_obj_add_val(mdoc, mroot, "linked_projects", lp_arr); diff --git a/tests/test_cross_repo.c b/tests/test_cross_repo.c index cb4ab0607..54c75cbd5 100644 --- a/tests/test_cross_repo.c +++ b/tests/test_cross_repo.c @@ -10,6 +10,7 @@ #include "pipeline/pipeline_internal.h" #include +#include #include #include #include @@ -401,6 +402,96 @@ TEST(cross_repo_named_pre768_target_fails_before_cleanup_issue2133) { PASS(); } +/* Render the "cross_repo" status object the MCP/UI surfaces attach for + * `project`, read from its store the way index_status does (query open). */ +static char *cross_repo_status_json(const cross_repo_fixture_t *fixture, const char *project) { + char path[512]; + if (!cross_repo_project_path(fixture, project, path, sizeof(path))) { + return NULL; + } + cbm_store_t *store = cbm_store_open_path_query(path); + if (!store) { + return NULL; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + char *json = NULL; + if (cbm_cross_repo_add_status_json(doc, root, store, project)) { + json = yyjson_mut_write(doc, 0, NULL); + } + yyjson_mut_doc_free(doc); + cbm_store_close(store); + return json; +} + +/* #2133 follow-up: an empty CROSS_* set could mean "never linked" or "linked, + * nothing matched" and no surface told them apart. The source store records + * each run; before any run the status says so explicitly. */ +TEST(cross_repo_status_distinguishes_never_run_from_ran_without_links) { + cross_repo_fixture_t fixture; + bool setup = cross_repo_fixture_begin(&fixture) && + cross_repo_create_project(&fixture, "status-src") && + cross_repo_create_project(&fixture, "status-api"); + if (!setup) { + cross_repo_fixture_end(&fixture); + FAIL("failed to seed status fixture"); + } + + char *before = cross_repo_status_json(&fixture, "status-src"); + const char *targets[] = {"status-api"}; + cbm_cross_repo_result_t result = cbm_cross_repo_match("status-src", targets, 1); + char *after = cross_repo_status_json(&fixture, "status-src"); + cross_repo_fixture_end(&fixture); + + bool before_never = before && strstr(before, "\"cross_repo\":{\"status\":\"never_run\"}"); + bool after_ran = + after && strstr(after, "\"status\":\"ran\"") && strstr(after, "\"outcome\":\"complete\"") && + strstr(after, "\"targets\":[\"status-api\"]") && strstr(after, "\"projects_scanned\":1") && + strstr(after, "\"total_cross_edges\":0") && strstr(after, "\"last_run_at\":\"20"); + free(before); + free(after); + cbm_cross_repo_result_free(&result); + + ASSERT_FALSE(result.failed); + ASSERT_TRUE(before_never); + ASSERT_TRUE(after_ran); + PASS(); +} + +/* #2133 follow-up: ["*"] skips a pre-#768 store (it cannot be linked until a + * reindex); the result must name it rather than silently covering fewer + * projects, and the recorded status counts it. */ +TEST(cross_repo_wildcard_reports_skipped_pre768_store) { + cross_repo_fixture_t fixture; + bool setup = cross_repo_fixture_begin(&fixture) && + cross_repo_seed_http_pair(&fixture, "skip-src", "skip-api", "/orders", "s") && + cross_repo_create_pre768_project(&fixture, "aa-pre768-store"); + if (!setup) { + cross_repo_fixture_end(&fixture); + FAIL("failed to seed skipped-store fixture"); + } + + const char *targets[] = {"*"}; + cbm_cross_repo_result_t result = cbm_cross_repo_match("skip-src", targets, 1); + char *status = cross_repo_status_json(&fixture, "skip-src"); + cross_repo_fixture_end(&fixture); + + bool named = result.skipped_count == 1 && result.skipped_projects && + strcmp(result.skipped_projects[0].project, "aa-pre768-store") == 0 && + strcmp(result.skipped_projects[0].reason, "pre_768_schema") == 0; + bool status_counts = + status && strstr(status, "\"skipped_projects\":1") && strstr(status, "\"targets\":[\"*\"]"); + free(status); + cbm_cross_repo_result_free(&result); + + ASSERT_FALSE(result.failed); + ASSERT_EQ(result.projects_scanned, 1); + ASSERT_TRUE(named); + ASSERT_TRUE(status_counts); + PASS(); +} + static bool cross_repo_seed_bounded_scan(const cross_repo_fixture_t *fixture, const char *source_project, const char *target_project) { enum { TEST_SCAN_ROWS = 4097 }; @@ -836,6 +927,8 @@ SUITE(cross_repo) { RUN_TEST(cross_repo_wildcard_keeps_projects_containing_internal_tokens); RUN_TEST(cross_repo_wildcard_skips_pre768_store_issue2133); RUN_TEST(cross_repo_named_pre768_target_fails_before_cleanup_issue2133); + RUN_TEST(cross_repo_status_distinguishes_never_run_from_ran_without_links); + RUN_TEST(cross_repo_wildcard_reports_skipped_pre768_store); RUN_TEST(cross_repo_scan_bound_counts_examined_rows_not_matches); RUN_TEST(cross_repo_propagates_delete_failure); RUN_TEST(cross_repo_failed_bidirectional_insert_is_not_counted); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 7a2219c5a..883812157 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -13033,6 +13033,102 @@ TEST(tool_cross_repo_dedupes_targets_before_scanning_and_counting) { PASS(); } +/* A store written before the #768 edges.local_name_gen column: readable, but + * refused by every read-write open until a reindex. */ +static bool mcp_cross_repo_create_pre768_store(const char *cache, const char *project) { + char db_path[CBM_SZ_1K]; + snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, project); + sqlite3 *db = NULL; + if (sqlite3_open(db_path, &db) != SQLITE_OK) { + sqlite3_close(db); + return false; + } + char sql[CBM_SZ_1K]; + snprintf(sql, sizeof(sql), + "CREATE TABLE projects(name TEXT PRIMARY KEY, indexed_at TEXT NOT NULL," + " root_path TEXT NOT NULL);" + "CREATE TABLE nodes(id INTEGER PRIMARY KEY AUTOINCREMENT, project TEXT NOT NULL," + " label TEXT NOT NULL, name TEXT NOT NULL, qualified_name TEXT NOT NULL," + " file_path TEXT DEFAULT '', start_line INTEGER DEFAULT 0," + " end_line INTEGER DEFAULT 0, properties TEXT DEFAULT '{}'," + " UNIQUE(project, qualified_name));" + "CREATE TABLE edges(id INTEGER PRIMARY KEY AUTOINCREMENT, project TEXT NOT NULL," + " source_id INTEGER NOT NULL, target_id INTEGER NOT NULL, type TEXT NOT NULL," + " properties TEXT DEFAULT '{}', UNIQUE(source_id, target_id, type));" + "INSERT INTO projects VALUES('%s', '2026-06-01T00:00:00Z', '/pre768');", + project); + bool ok = sqlite3_exec(db, sql, NULL, NULL, NULL) == SQLITE_OK; + sqlite3_close(db); + return ok; +} + +/* #2133 follow-up, both user-visible halves: index_status tells "never run" + * from "ran" via a `cross_repo` object, and a ["*"] run names the legacy + * store it skipped (with the reindex hint) instead of silently linking fewer + * projects. */ +TEST(tool_cross_repo_status_and_wildcard_skipped_projects) { + char cache[256]; + snprintf(cache, sizeof(cache), "%s/cbm-mcp-cross-status-XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(cache)); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + + char *source_project = cbm_project_name_from_path(cache); + ASSERT_NOT_NULL(source_project); + const char *target_project = "cross-status-target"; + const char *legacy_project = "cross-status-pre768"; + ASSERT_TRUE(mcp_cross_repo_seed_http_match(cache, source_project, target_project, cache)); + ASSERT_TRUE(mcp_cross_repo_create_pre768_store(cache, legacy_project)); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + ASSERT_TRUE(cbm_mcp_server_set_session_context(srv, cache, NULL)); + + char status_args[CBM_SZ_1K]; + snprintf(status_args, sizeof(status_args), "{\"project\":\"%s\",\"format\":\"json\"}", + source_project); + char *before = cbm_mcp_handle_tool(srv, "index_status", status_args); + bool before_never = + response_contains_json_fragment(before, "\"cross_repo\":{\"status\":\"never_run\"}"); + + char args[CBM_SZ_2K]; + snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"cross-repo-intelligence\"," + "\"target_projects\":[\"*\"]}", + cache); + char *run = cbm_mcp_handle_tool(srv, "index_repository", args); + bool run_ok = run && strstr(run, "\"isError\":true") == NULL && + response_contains_json_fragment(run, "\"total_cross_edges\":1"); + bool skipped_named = response_contains_json_fragment( + run, "\"skipped_projects\":[{\"name\":\"cross-status-pre768\"," + "\"reason\":\"pre_768_schema\",\"hint\":\"reindex this project\"}]"); + + char *after = cbm_mcp_handle_tool(srv, "index_status", status_args); + bool after_ran = response_contains_json_fragment(after, "\"status\":\"ran\"") && + response_contains_json_fragment(after, "\"total_cross_edges\":1") && + response_contains_json_fragment(after, "\"skipped_projects\":1"); + + free(before); + free(run); + free(after); + cbm_mcp_server_free(srv); + cleanup_project_db(cache, source_project); + cleanup_project_db(cache, target_project); + cleanup_project_db(cache, legacy_project); + free(source_project); + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + cbm_rmdir(cache); + + ASSERT_TRUE(before_never); + ASSERT_TRUE(run_ok); + ASSERT_TRUE(skipped_named); + ASSERT_TRUE(after_ran); + PASS(); +} + /* `name` is the documented index project-name override and must identify the * cross-repo source too. Deriving from repo_path here makes custom-named * projects impossible to rescan even though ordinary indexing created them. */ @@ -21212,6 +21308,7 @@ SUITE(mcp_mutation_guard) { RUN_TEST(tool_cross_repo_checks_cancellation_after_acquiring_leases); RUN_TEST(tool_cross_repo_missing_inputs_fail_without_creating_ghost_databases); RUN_TEST(tool_cross_repo_dedupes_targets_before_scanning_and_counting); + RUN_TEST(tool_cross_repo_status_and_wildcard_skipped_projects); RUN_TEST(tool_cross_repo_honors_source_name_override); RUN_TEST(tool_corrupt_store_cleanup_guard_is_balanced_and_not_nested); RUN_TEST(tool_corrupt_store_cleanup_guard_denial_preserves_db_and_wal); From 6eb33534f68b29f36c012b18a0236b1bd6b300c4 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 25 Sep 2026 18:37:16 +0200 Subject: [PATCH 04/34] fix(daemon): explain foreign-owned cache ancestors and warn on network filesystems (#1717) A cache root whose resolved path runs through a directory owned by another non-root account (typical on shared/HPC filesystems where ~/.cache is a symlink into a shared tree) is refused by the private-directory walk. The refusal itself is correct: the owner of an ancestor can rename or replace everything below it after the walk, so it could redirect the cache. But posix_directory_transition_secure() set no validation detail, so the caller printed "ancestry component validation failed (errno 17)" -- a stale EEXIST from mkdirat() that named neither the directory nor the reason. - The refusal now names the directory, its owner uid and account name, the caller's uid, and the remedy (a location with a self/root-owned ancestry, e.g. via CBM_CACHE_DIR). The path is shortened from the left so the remedy always fits the 384-byte detail buffer. - On Linux the account name is read from /etc/passwd, not getpwuid(). The release binary is statically linked against glibc, and getpwuid() there loads the host's NSS modules at runtime, which can crash on a glibc version mismatch (a real risk on sssd/LDAP cluster hosts). Accounts that exist only in a directory service are shown by uid. - New advice-only check: on Linux, a cache root on NFS, CIFS, SMB2, Lustre or GPFS (statfs f_type) logs one daemon.cache_root_network_fs warning per process. SQLite WAL is unreliable on network filesystems. Nothing is refused. - docs/CONFIGURATION.md gains a "Shared / HPC filesystems" section. The ownership policy is unchanged, and there is still no override (same stance as #1830). Test seams: one seam reports a directory (matched by dev/ino) as foreign-owned, and another supplies the statfs magic. The refusal, the message and the once-per-process warning are therefore testable without root and on every platform. Refs #1717 Signed-off-by: Martin Vogel --- docs/CONFIGURATION.md | 27 ++++++ src/daemon/ipc.c | 209 +++++++++++++++++++++++++++++++++++++++- src/daemon/ipc.h | 17 ++++ src/main.c | 2 + tests/test_daemon_ipc.c | 207 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 458 insertions(+), 4 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 024f37e93..aa17e5005 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -235,6 +235,33 @@ be indexed and later returned. And the credential list is a denylist, so it raises the cost of a mistake rather than closing the class — a directory it does not name is permitted. +### Shared / HPC filesystems + +On clusters and other shared machines, home directories often live on a network +filesystem (NFS, Lustre, GPFS, SMB/CIFS), and `~/.cache` is sometimes a symlink +into a shared tree. Three rules keep CBM working there: + +- **Put the cache and the rendezvous on local storage.** Set `CBM_CACHE_DIR` (indexes, + `_config.db`) and, if needed, `CBM_RUNTIME_DIR` (the daemon rendezvous) to a + local disk you own, for example `/local/scratch/$USER/cbm` or `/tmp/$USER-cbm`. + Set the same values in your MCP client's environment and your shell. +- **Do not route the cache into another account's tree.** A symlinked `~/.cache` is + resolved and allowed, but every directory on the *resolved* path must be owned + by you or by root. If a parent such as `/shared/fs` belongs to another account, + that account can rename or replace anything below it, so CBM refuses the path + and names the directory and its owner: + + ```text + ... (cache-private) - '/shared/fs' is owned by uid 1234 (alice), not by you (uid 1000) or root; ... + ``` + + This check has no override. Choose a location with a safe ancestry instead. +- **Avoid network filesystems for the cache.** The indexes are SQLite databases in + WAL mode, which needs shared-memory locking that network filesystems do not + provide reliably. On Linux, CBM logs one `daemon.cache_root_network_fs` warning + at startup when the cache root is on NFS, Lustre, GPFS, CIFS or SMB2. The cache + still works there, but it can be slow, and a database can be corrupted. + ## 5. Agent and Editor Integration Files The `install` command can also write MCP entries and instruction blocks into agent/editor config files such as Claude Code, Codex, Gemini, VS Code, Cursor, Zed, and others. diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index eb87e43be..0ea819542 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -21,6 +21,9 @@ #include #include #include +#if defined(__linux__) +#include +#endif enum { CBM_DAEMON_IPC_SEND_TIMEOUT_MS = 5000, @@ -54,6 +57,76 @@ const char *cbm_daemon_ipc_validation_detail(void) { return ipc_validation_detail_buffer; } +/* #1717: a cache root on a network filesystem (a shared HPC home, an NFS or + * SMB mount) is ACCEPTED -- this is advice, never a refusal. SQLite's WAL mode + * relies on shared-memory locking that network filesystems do not provide + * reliably, so indexes there can be slow or corrupt. Detection is statfs + * f_type, Linux only; other platforms never warn. */ +static const char *ipc_network_fs_name(unsigned long magic) { + static const struct { + unsigned long magic; + const char *name; + } kinds[] = { + {0x6969UL, "NFS"}, {0xFF534D42UL, "CIFS"}, {0xFE534D42UL, "SMB2"}, + {0x0BD00BD0UL, "Lustre"}, {0x47504653UL, "GPFS"}, + }; + for (size_t index = 0; index < sizeof(kinds) / sizeof(kinds[0]); index++) { + if (kinds[index].magic == magic) { + return kinds[index].name; + } + } + return NULL; +} + +static atomic_bool g_ipc_network_cache_warned; +#ifdef CBM_ENABLE_TEST_SEAMS +static bool g_ipc_fs_magic_override_active; +static unsigned long g_ipc_fs_magic_override; +void cbm_daemon_ipc_set_fs_magic_for_test(bool active, unsigned long magic) { + g_ipc_fs_magic_override_active = active; + g_ipc_fs_magic_override = magic; + atomic_store(&g_ipc_network_cache_warned, false); +} +#endif + +static bool ipc_path_fs_magic(const char *path, unsigned long *magic_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_ipc_fs_magic_override_active) { + *magic_out = g_ipc_fs_magic_override; + return true; + } +#endif +#if defined(__linux__) + struct statfs status; + if (statfs(path, &status) != 0) { + return false; + } + /* f_type is a signed word on some ABIs; the magics are 32-bit. */ + *magic_out = (unsigned long)status.f_type & 0xFFFFFFFFUL; + return true; +#else + (void)path; + (void)magic_out; + return false; +#endif +} + +bool cbm_daemon_ipc_warn_if_network_cache_root(const char *cache_root) { + unsigned long magic = 0; + const char *kind = NULL; + if (!cache_root || !cache_root[0] || !ipc_path_fs_magic(cache_root, &magic) || + !(kind = ipc_network_fs_name(magic))) { + return false; + } + if (atomic_exchange(&g_ipc_network_cache_warned, true)) { + return false; /* one warning per process */ + } + cbm_log_warn("daemon.cache_root_network_fs", "path", cache_root, "fs", kind, "reason", + "sqlite_wal_unreliable_on_network_fs", "hint", + "set CBM_CACHE_DIR to local storage (docs/CONFIGURATION.md)"); + return true; +} + static cbm_daemon_ipc_listen_failure_t ipc_listen_failure; /* The failure recorder and its reset are only reached from the POSIX socket @@ -401,6 +474,9 @@ int cbm_daemon_ipc_wait_pending(const cbm_ipc_pending_ops_t *ops, uint32_t timeo #include #include #include +#if !defined(__linux__) +#include +#endif #include #include #include @@ -1788,14 +1864,138 @@ static bool posix_directory_parent_secure(int directory_fd) { * never chmod'd or ACL-rewritten. The euid-only enforcement that stops a * squatted private directory lives in the created/final/snapshot checks, not * here — this only walks ancestors. */ -static bool posix_directory_transition_secure(int parent_fd, int child_fd) { +#ifdef CBM_ENABLE_TEST_SEAMS +/* #1717 seam: report one directory (by dev/ino) as owned by a foreign uid, so + * the foreign-owned-ancestor refusal and its diagnostic are testable without + * root (an unprivileged test cannot chown a directory to another account). */ +static bool g_posix_foreign_owner_active; +static dev_t g_posix_foreign_owner_dev; +static ino_t g_posix_foreign_owner_ino; +static uid_t g_posix_foreign_owner_uid; +bool cbm_daemon_ipc_posix_set_foreign_owned_dir_for_test(const char *directory_path, + unsigned long foreign_uid) { + struct stat status; + g_posix_foreign_owner_active = false; + if (!directory_path || lstat(directory_path, &status) != 0 || !S_ISDIR(status.st_mode)) { + return false; + } + g_posix_foreign_owner_dev = status.st_dev; + g_posix_foreign_owner_ino = status.st_ino; + g_posix_foreign_owner_uid = (uid_t)foreign_uid; + g_posix_foreign_owner_active = true; + return true; +} +void cbm_daemon_ipc_posix_clear_foreign_owned_dir_for_test(void) { + g_posix_foreign_owner_active = false; +} +#endif + +static uid_t posix_directory_observed_owner(const struct stat *status) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_posix_foreign_owner_active && status->st_dev == g_posix_foreign_owner_dev && + status->st_ino == g_posix_foreign_owner_ino) { + return g_posix_foreign_owner_uid; + } +#endif + return status->st_uid; +} + +enum { POSIX_OWNER_NAME_CAP = 33, POSIX_PASSWD_LINE_CAP = 1024 }; + +/* The account name for uid, or "" when it cannot be named. Diagnostic only. + * + * Linux reads /etc/passwd directly instead of calling getpwuid(): the Linux + * release binary is statically linked against glibc, where getpwuid() loads the + * HOST's NSS modules at runtime -- a version-mismatch crash hazard exactly on + * the sssd/LDAP hosts (#1717's shared HPC filesystems) this message serves. + * A directory-service account is then shown by uid only. */ +static void posix_owner_name(uid_t uid, char out[POSIX_OWNER_NAME_CAP]) { + out[0] = '\0'; +#if defined(__linux__) + FILE *passwd = cbm_fopen("/etc/passwd", "r"); + if (!passwd) { + return; + } + char line[POSIX_PASSWD_LINE_CAP]; + while (fgets(line, sizeof(line), passwd)) { + char *name_end = strchr(line, ':'); + char *password_end = name_end ? strchr(name_end + 1, ':') : NULL; + if (!password_end || name_end == line) { + continue; + } + char *uid_end = NULL; + errno = 0; + unsigned long line_uid = strtoul(password_end + 1, &uid_end, 10); + if (errno == 0 && uid_end != password_end + 1 && *uid_end == ':' && + line_uid == (unsigned long)uid) { + (void)snprintf(out, POSIX_OWNER_NAME_CAP, "%.*s", (int)(name_end - line), line); + break; + } + } + (void)fclose(passwd); +#else + struct passwd entry; + struct passwd *found = NULL; + char storage[POSIX_PASSWD_LINE_CAP]; + if (getpwuid_r(uid, &entry, storage, sizeof(storage), &found) == 0 && found && found->pw_name) { + (void)snprintf(out, POSIX_OWNER_NAME_CAP, "%s", found->pw_name); + } +#endif +} + +/* The foreign-owned-ancestor refusal. The remedy is the part the reader acts + * on, so it must survive the fixed detail buffer: the path is shortened from + * the LEFT ("...") to fit, which keeps the refusing component at its end. */ +static void posix_foreign_owner_detail_set(const char *child_path, uid_t owner) { + char name[POSIX_OWNER_NAME_CAP]; + posix_owner_name(owner, name); + char owner_text[64]; + if (name[0]) { + (void)snprintf(owner_text, sizeof(owner_text), "uid %ld (%s)", (long)owner, name); + } else { + (void)snprintf(owner_text, sizeof(owner_text), "uid %ld", (long)owner); + } + char tail[sizeof(ipc_validation_detail_buffer)]; + int tail_length = snprintf(tail, sizeof(tail), + "' is owned by %s, not by you (uid %ld) or root; that account " + "controls this path. Use a location whose parent directories are " + "yours or root's (e.g. set CBM_CACHE_DIR)", + owner_text, (long)geteuid()); + if (tail_length < 0) { + return; + } + size_t used = (size_t)tail_length + 1; /* the opening quote */ + size_t room = used < sizeof(ipc_validation_detail_buffer) + ? sizeof(ipc_validation_detail_buffer) - 1 - used + : 0; + size_t path_length = strlen(child_path); + const char *ellipsis = ""; + const char *shown = child_path; + if (path_length > room) { + ellipsis = "..."; + shown = child_path + path_length - (room > 3 ? room - 3 : 0); + } + ipc_validation_detail_set("'%s%s%s", ellipsis, shown, tail); +} + +static bool posix_directory_transition_secure(int parent_fd, int child_fd, const char *child_path) { struct stat parent; struct stat child; if (parent_fd < 0 || child_fd < 0 || !posix_directory_parent_secure(parent_fd) || fstat(parent_fd, &parent) != 0 || fstat(child_fd, &child) != 0 || !S_ISDIR(parent.st_mode) || !S_ISDIR(child.st_mode) || - !posix_directory_ancestor_owner_trusted(parent.st_uid) || - !posix_directory_ancestor_owner_trusted(child.st_uid)) { + !posix_directory_ancestor_owner_trusted(posix_directory_observed_owner(&parent))) { + return false; + } + uid_t child_owner = posix_directory_observed_owner(&child); + if (!posix_directory_ancestor_owner_trusted(child_owner)) { + /* #1717: this refusal used to set no detail, so the caller printed + * "ancestry component validation failed (errno 17)" -- a stale EEXIST + * from the mkdirat() on the existing component, naming neither the + * directory nor the reason. The policy is unchanged: an account that + * owns an ancestor can rename or replace everything below it after + * this walk, so it could redirect the private directory. Name it. */ + posix_foreign_owner_detail_set(child_path, child_owner); return false; } return true; @@ -1870,7 +2070,8 @@ static int private_directory_tree_open(const char *directory_path) { : -1; struct stat status; ok = next_fd >= 0 && fd_set_cloexec(next_fd) && fstat(next_fd, &status) == 0 && - S_ISDIR(status.st_mode) && posix_directory_transition_secure(current_fd, next_fd); + S_ISDIR(status.st_mode) && + posix_directory_transition_secure(current_fd, next_fd, path); if (ok && created) { ok = status.st_uid == geteuid() && fchmod(next_fd, 0700) == 0 && cbm_macos_extended_acl_fd_clear(next_fd) && diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index effe6a852..f8195f3d5 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -92,6 +92,17 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path); * append it to their error messages; policy decisions never read it. */ const char *cbm_daemon_ipc_validation_detail(void); +/* #1717: advice, never a refusal. When cache_root lives on a network + * filesystem (Linux statfs: NFS, CIFS, SMB2, Lustre, GPFS), log one + * `daemon.cache_root_network_fs` warning per process pointing at local + * storage. Returns true only when this call emitted that warning. */ +bool cbm_daemon_ipc_warn_if_network_cache_root(const char *cache_root); +#ifdef CBM_ENABLE_TEST_SEAMS +/* Report `magic` as the cache root's statfs f_type on every platform (active), + * or restore real detection; either way re-arm the once-per-process warning. */ +void cbm_daemon_ipc_set_fs_magic_for_test(bool active, unsigned long magic); +#endif + /* This process's most recent listener-publication failure: the stage that * refused, the errno the failing step reported (0 when the step reported * none), and the artifact path it was operating on ("" when no path applies). @@ -127,6 +138,12 @@ bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid); * so they can be tested without a chown-able overflow-owned directory. */ void cbm_daemon_ipc_posix_set_ancestor_overflow_uid_for_test(bool active, unsigned long overflow_uid); +/* #1717 seam (POSIX): report the directory at directory_path (matched by + * dev/ino) as owned by foreign_uid during the ancestor walk. False when the + * path is not a directory. Clear restores real ownership. */ +bool cbm_daemon_ipc_posix_set_foreign_owned_dir_for_test(const char *directory_path, + unsigned long foreign_uid); +void cbm_daemon_ipc_posix_clear_foreign_owned_dir_for_test(void); bool cbm_daemon_ipc_posix_uid_map_is_single_uid_for_test(const char *uid_map, unsigned long euid); bool cbm_daemon_ipc_posix_ancestor_stat_ok_for_test(unsigned long owner, unsigned int mode, unsigned long euid, bool overflow_active, diff --git a/src/main.c b/src/main.c index fd3370c04..3f179fffe 100644 --- a/src/main.c +++ b/src/main.c @@ -1455,6 +1455,8 @@ static main_build_identity_status_t main_build_identity(cbm_daemon_build_identit if (!cbm_daemon_ipc_private_directory_secure(canonical_cache)) { return MAIN_BUILD_IDENTITY_CACHE_PRIVATE; } + /* #1717: advice only -- a network-filesystem cache root is accepted. */ + (void)cbm_daemon_ipc_warn_if_network_cache_root(canonical_cache); /* Every cache consumer in this process must use the exact path whose * fingerprint joins the account-wide cohort. Keeping an original symlink * spelling in the environment would let a later retarget move storage diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 97257506c..35ad5433c 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #ifdef __linux__ #include /* #1830 userns real smoke: unshare(CLONE_NEWUSER). */ @@ -5274,6 +5275,147 @@ TEST(daemon_ipc_posix_wsl_drvfs_refusal_names_remedy_issue1687) { PASS(); } +/* #1717: an ancestor owned by ANOTHER non-root account (a shared filesystem + * whose parent directory belongs to a colleague) is refused -- that account can + * rename or replace everything below it, so the policy stays. What was wrong is + * the message: the refusal set no detail, and the caller printed + * "ancestry component validation failed (errno 17)", a stale EEXIST from + * mkdirat() that named neither the directory nor the reason. The reporter + * could not tell which directory, or why. The refusal must name the refusing + * directory, its owner uid, and the remedy. The seam reports one directory as + * foreign-owned because an unprivileged test cannot chown to another account. */ +enum { IPC_TEST_FOREIGN_MAX_DEPTH = 16 }; + +typedef struct { + bool paths_ok; + bool seam_set; + bool refused; + bool cache_created; + bool secured_without_seam; + size_t foreign_path_length; + char detail[512]; +} ipc_test_foreign_result_t; + +/* Build //shared-fs, mark shared-fs as owned by + * foreign_uid through the seam, and ask for <...>/shared-fs/cache. Deeper + * chains produce a path long enough that the detail must be shortened. */ +static void ipc_test_foreign_ancestor_run(const char *tag, int depth, unsigned long foreign_uid, + ipc_test_foreign_result_t *result) { + char parent[TEST_PATH_CAP]; + char chain[IPC_TEST_FOREIGN_MAX_DEPTH + 1][TEST_PATH_CAP]; + char cache[TEST_PATH_CAP]; + int made = 0; + memset(result, 0, sizeof(*result)); + result->paths_ok = depth <= IPC_TEST_FOREIGN_MAX_DEPTH && ipc_test_parent_new(parent, tag); + const char *base = parent; + for (int level = 0; result->paths_ok && level <= depth; level++) { + const char *leaf = level == depth ? "shared-fs" : "level-component-24-chars"; + int w = snprintf(chain[level], sizeof(chain[level]), "%s/%s", base, leaf); + result->paths_ok = w > 0 && w < (int)sizeof(chain[level]) && mkdir(chain[level], 0755) == 0; + made += result->paths_ok ? 1 : 0; + base = chain[level]; + } + if (result->paths_ok) { + int c = snprintf(cache, sizeof(cache), "%s/cache", chain[depth]); + result->paths_ok = c > 0 && c < (int)sizeof(cache); + result->foreign_path_length = strlen(chain[depth]); + } + if (result->paths_ok) { + result->seam_set = + cbm_daemon_ipc_posix_set_foreign_owned_dir_for_test(chain[depth], foreign_uid); + } + if (result->seam_set) { + result->refused = !cbm_daemon_ipc_private_directory_secure(cache); + const char *why = cbm_daemon_ipc_validation_detail(); + (void)snprintf(result->detail, sizeof(result->detail), "%s", why ? why : ""); + struct stat probe; + result->cache_created = lstat(cache, &probe) == 0; + cbm_daemon_ipc_posix_clear_foreign_owned_dir_for_test(); + /* Control: with real ownership the same path is accepted, so the + * refusal above is the foreign owner and nothing else. */ + result->secured_without_seam = cbm_daemon_ipc_private_directory_secure(cache); + (void)rmdir(cache); + } + cbm_daemon_ipc_posix_clear_foreign_owned_dir_for_test(); + while (made > 0) { + (void)rmdir(chain[--made]); + } + if (depth <= IPC_TEST_FOREIGN_MAX_DEPTH) { + ipc_test_remove_flat_dir(parent); + } +} + +TEST(daemon_ipc_posix_foreign_owned_ancestor_refusal_names_directory_issue1717) { + /* A uid no account resolves to: the detail names the uid alone. */ + unsigned long foreign_uid = (unsigned long)geteuid() + 4242UL; + ASSERT_TRUE(getpwuid((uid_t)foreign_uid) == NULL); /* precondition */ + ipc_test_foreign_result_t result; + ipc_test_foreign_ancestor_run("posix-foreign-ancestor", 0, foreign_uid, &result); + char owner_text[64]; + (void)snprintf(owner_text, sizeof(owner_text), "owned by uid %lu,", foreign_uid); + + ASSERT_TRUE(result.paths_ok); + ASSERT_TRUE(result.seam_set); + ASSERT_TRUE(result.refused); + ASSERT_TRUE(!result.cache_created); /* nothing is created below a foreign ancestor */ + ASSERT_NOT_NULL(strstr(result.detail, "/shared-fs'")); + ASSERT_NOT_NULL(strstr(result.detail, owner_text)); + ASSERT_NOT_NULL(strstr(result.detail, "CBM_CACHE_DIR")); + ASSERT_TRUE(strstr(result.detail, "errno") == NULL); + ASSERT_TRUE(result.secured_without_seam); + PASS(); +} + +/* #1717: the owner is named when the account resolves. uid 1 is a local system + * account in /etc/passwd on macOS and on the Linux images we test ("daemon" on + * both; Alpine calls it "bin"), so the expected name is taken from the same + * local database rather than hardcoded. */ +TEST(daemon_ipc_posix_foreign_owned_ancestor_names_owner_account_issue1717) { + const unsigned long named_uid = 1; + struct passwd *expected = getpwuid((uid_t)named_uid); + if (!expected || !expected->pw_name || !expected->pw_name[0]) { + SKIP_PLATFORM("uid 1 has no local account name on this system"); + } + char owner_text[96]; + (void)snprintf(owner_text, sizeof(owner_text), "owned by uid %lu (%s),", named_uid, + expected->pw_name); + ipc_test_foreign_result_t result; + ipc_test_foreign_ancestor_run("posix-foreign-named", 0, named_uid, &result); + + ASSERT_TRUE(result.paths_ok); + ASSERT_TRUE(result.seam_set); + ASSERT_TRUE(result.refused); + ASSERT_NOT_NULL(strstr(result.detail, owner_text)); + ASSERT_NOT_NULL(strstr(result.detail, "/shared-fs'")); + ASSERT_TRUE(result.secured_without_seam); + PASS(); +} + +/* #1717: the detail buffer is fixed (384 bytes). A long path must not push the + * remedy out: the path is shortened from the left, so the refusing component + * and the whole remedy survive. */ +TEST(daemon_ipc_posix_foreign_owned_ancestor_long_path_keeps_remedy_issue1717) { + const unsigned long named_uid = 1; + ipc_test_foreign_result_t result; + ipc_test_foreign_ancestor_run("posix-foreign-long", IPC_TEST_FOREIGN_MAX_DEPTH, named_uid, + &result); + static const char remedy[] = + "Use a location whose parent directories are yours or root's (e.g. set CBM_CACHE_DIR)"; + + ASSERT_TRUE(result.paths_ok); + ASSERT_TRUE(result.foreign_path_length > 384); /* the input really overflows */ + ASSERT_TRUE(result.refused); + ASSERT_TRUE(strlen(result.detail) < 384); + ASSERT_TRUE(strncmp(result.detail, "'...", 4) == 0); + ASSERT_NOT_NULL(strstr(result.detail, "/shared-fs' is owned by uid 1")); + size_t detail_length = strlen(result.detail); + ASSERT_TRUE(detail_length >= sizeof(remedy) - 1); + ASSERT_STR_EQ(result.detail + detail_length - (sizeof(remedy) - 1), remedy); + PASS(); +} +#endif + +#ifdef CBM_ENABLE_TEST_SEAMS /* #1830: /proc/self/uid_map single-uid detection. A single-uid map is exactly * one line " 1" whose inside id is our euid; anything else — * the init map, a count other than 1, a foreign inside id, extra lines, or junk @@ -5496,6 +5638,65 @@ static void ipc_test_log_capture_sink(const char *line) { ipc_test_log_capture[ipc_test_log_capture_used] = '\0'; } +#ifdef CBM_ENABLE_TEST_SEAMS +/* #1717: a cache root on a network filesystem (shared HPC home, NFS/SMB mount) + * is ACCEPTED, but SQLite WAL is unreliable there, so startup logs ONE warning + * with the local-storage advice. The seam supplies the statfs f_type on every + * platform (real detection is Linux-only), so the mapping and the + * once-per-process rule are bound deterministically here. */ +static bool ipc_test_network_warning_emitted(unsigned long magic, const char *fs_name) { + ipc_test_log_capture_used = 0; + ipc_test_log_capture[0] = '\0'; + cbm_daemon_ipc_set_fs_magic_for_test(true, magic); + cbm_log_set_sink_ex(ipc_test_log_capture_sink, CBM_LOG_SINK_REPLACE); + bool warned = cbm_daemon_ipc_warn_if_network_cache_root("/shared/fs/me/.cache/cbm"); + cbm_log_set_sink(NULL); + char expected[32]; + (void)snprintf(expected, sizeof(expected), "fs=%s", fs_name); + const char *line = strstr(ipc_test_log_capture, "daemon.cache_root_network_fs"); + return warned && line && strstr(line, expected) && strstr(line, "CBM_CACHE_DIR") && + strstr(line, "path=/shared/fs/me/.cache/cbm"); +} + +TEST(daemon_ipc_network_cache_root_warns_once_issue1717) { + CBMLogLevel saved_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_WARN); + bool nfs = ipc_test_network_warning_emitted(0x6969UL, "NFS"); + bool cifs = ipc_test_network_warning_emitted(0xFF534D42UL, "CIFS"); + bool smb2 = ipc_test_network_warning_emitted(0xFE534D42UL, "SMB2"); + bool lustre = ipc_test_network_warning_emitted(0x0BD00BD0UL, "Lustre"); + bool gpfs = ipc_test_network_warning_emitted(0x47504653UL, "GPFS"); + + /* Once per process: a second startup check stays silent. */ + cbm_daemon_ipc_set_fs_magic_for_test(true, 0x6969UL); + bool first = cbm_daemon_ipc_warn_if_network_cache_root("/nfs/cache"); + bool second = cbm_daemon_ipc_warn_if_network_cache_root("/nfs/cache"); + + /* Local filesystems (ext4, xfs, tmpfs) never warn. */ + cbm_daemon_ipc_set_fs_magic_for_test(true, 0xEF53UL); + bool ext4 = cbm_daemon_ipc_warn_if_network_cache_root("/home/me/.cache/cbm"); + cbm_daemon_ipc_set_fs_magic_for_test(true, 0x58465342UL); + bool xfs = cbm_daemon_ipc_warn_if_network_cache_root("/home/me/.cache/cbm"); + cbm_daemon_ipc_set_fs_magic_for_test(true, 0x01021994UL); + bool tmpfs = cbm_daemon_ipc_warn_if_network_cache_root("/home/me/.cache/cbm"); + + cbm_daemon_ipc_set_fs_magic_for_test(false, 0); + cbm_log_set_level(saved_level); + + ASSERT_TRUE(nfs); + ASSERT_TRUE(cifs); + ASSERT_TRUE(smb2); + ASSERT_TRUE(lustre); + ASSERT_TRUE(gpfs); + ASSERT_TRUE(first); + ASSERT_TRUE(!second); + ASSERT_TRUE(!ext4); + ASSERT_TRUE(!xfs); + ASSERT_TRUE(!tmpfs); + PASS(); +} +#endif + /* #1828: a full /tmp made every daemon start die at pending publication, and * the only durable trace was `daemon.ipc.listen_failed stage=pending_publication` * -- no syscall, no errno, no path. The reporter needed hours (and a wrong @@ -5599,6 +5800,9 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_posix_world_writable_ancestor_still_refused_issue1537); #ifdef CBM_ENABLE_TEST_SEAMS RUN_TEST(daemon_ipc_posix_wsl_drvfs_refusal_names_remedy_issue1687); + RUN_TEST(daemon_ipc_posix_foreign_owned_ancestor_refusal_names_directory_issue1717); + RUN_TEST(daemon_ipc_posix_foreign_owned_ancestor_names_owner_account_issue1717); + RUN_TEST(daemon_ipc_posix_foreign_owned_ancestor_long_path_keeps_remedy_issue1717); RUN_TEST(daemon_ipc_posix_uid_map_single_uid_parse_issue1830); RUN_TEST(daemon_ipc_posix_overflow_ancestor_tolerated_only_in_single_uid_ns_issue1830); RUN_TEST(daemon_ipc_posix_overflow_uid_is_never_cached_issue1830); @@ -5633,6 +5837,9 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_posix_rejects_non_socket_and_symlink_endpoints); #ifndef _WIN32 RUN_TEST(daemon_ipc_listen_failure_names_errno_and_path); +#ifdef CBM_ENABLE_TEST_SEAMS + RUN_TEST(daemon_ipc_network_cache_root_warns_once_issue1717); +#endif #endif #endif } From bc66208be6a240a684ff1b4bcf6dab60dd048851 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 1 Oct 2026 19:11:51 +0200 Subject: [PATCH 05/34] style(cross-repo): narrow the scope of the status edge total CI's cppcheck flags `total` in the cross-repo status writer (variableScope): it is only read inside the `len < cap` branch. Move the declaration into that branch. No behaviour change. Signed-off-by: Martin Vogel --- src/pipeline/pass_cross_repo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline/pass_cross_repo.c b/src/pipeline/pass_cross_repo.c index f645db349..09384a8e9 100644 --- a/src/pipeline/pass_cross_repo.c +++ b/src/pipeline/pass_cross_repo.c @@ -1361,9 +1361,9 @@ static void cr_record_last_run(cbm_store_t *src_store, const char *project, cons len += (size_t)snprintf(json + len, cap - len, "%s\"%s\"", i ? "," : "", targets[i] ? targets[i] : ""); } - int total = result->http_edges + result->async_edges + result->channel_edges + - result->grpc_edges + result->graphql_edges + result->trpc_edges; if (len < cap) { + int total = result->http_edges + result->async_edges + result->channel_edges + + result->grpc_edges + result->graphql_edges + result->trpc_edges; len += (size_t)snprintf(json + len, cap - len, "],\"projects_scanned\":%d,\"total_cross_edges\":%d," "\"skipped_projects\":%d}", From 402dd7c59b7287416361edac782fd642ac027cc6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 1 Oct 2026 19:59:02 +0200 Subject: [PATCH 06/34] fix(artifact): create .gitattributes through the UTF-8 path layer Follow-up to the #1171 fix. Exporting an artifact from a repository under a CJK path on Windows wrote graph.db.zst and artifact.json, but the .codebase-memory/.gitattributes file carrying the merge=ours protection was silently missing. Root cause: ensure_gitattributes() created the file with the raw POSIX open(O_WRONLY | O_CREAT | O_EXCL). On Windows that is the ANSI CRT, so a repo path with characters outside the active code page fails with ENOENT. The failure was only logged, and the warning itself was malformed: it passed printf-style arguments to the key/value logger, which printed the literal "msg=artifact.gitattributes.open_path=%s_err=%s =". Fix: create the file with cbm_fopen(path, "wbx") (_wfopen on Windows). The "x" mode keeps the O_CREAT | O_EXCL create-only-if-absent semantics, so an existing (possibly user-edited) .gitattributes is never rewritten. Binary mode writes the same LF bytes on every platform. The warning now uses the logger's key/value form (path=..., err=...). Test: artifact::artifact_roundtrip_non_ascii_paths now reads .gitattributes back from the CJK repo through cbm_fopen and asserts the merge=ours line. It also replaces the file with user content, exports again, and asserts the content is unchanged. On the Windows VM the assertion failed 3 of 3 without the fix and passes 3 of 3 with it. The no-overwrite check fails if the mode loses its "x". Refs #1171 Signed-off-by: Martin Vogel --- src/pipeline/artifact.c | 43 +++++++++++++++++++++-------------------- tests/test_artifact.c | 39 +++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/pipeline/artifact.c b/src/pipeline/artifact.c index 9594c8a09..b1a1ac6c7 100644 --- a/src/pipeline/artifact.c +++ b/src/pipeline/artifact.c @@ -772,30 +772,31 @@ static void ensure_gitattributes(const char *repo_path) { char ga_path[CBM_SZ_4K]; artifact_path(ga_path, sizeof(ga_path), repo_path, ".gitattributes"); - /* Atomic create-only-if-absent: O_EXCL closes the TOCTOU window - * between checking existence and writing. If the file exists, open - * fails with EEXIST and we leave it untouched. */ - int fd = open(ga_path, O_WRONLY | O_CREAT | O_EXCL, 0644); - if (fd < 0) { - if (errno != EEXIST) { - cbm_log_warn("artifact.gitattributes.open path=%s err=%s", ga_path, strerror(errno)); + /* Atomic create-only-if-absent: the "x" mode (O_CREAT|O_EXCL) closes the + * TOCTOU window between checking existence and writing. If the file + * exists, the open fails with EEXIST and we leave it untouched. + * cbm_fopen, not open(): the ANSI CRT open() cannot create the file under + * a repo path outside the active code page on Windows, which silently + * dropped the merge=ours protection there (#1171). */ + errno = 0; + FILE *fp = cbm_fopen(ga_path, "wbx"); + if (!fp) { + int open_errno = errno; + if (open_errno != EEXIST) { + cbm_log_warn("artifact.gitattributes.open", "path", ga_path, "err", + strerror(open_errno)); } /* fall through to merge driver setup either way */ } else { - FILE *fp = fdopen(fd, "w"); - if (fp) { - /* Order matters: attributes apply left to right and the `binary` - * macro expands to `-diff -merge -text`, so a trailing `binary` - * unsets `merge=ours` and the conflict prevention this file - * exists for never engages. The macro must come first. */ - (void)fputs("# Auto-generated by codebase-memory-mcp\n" - "# Prevent merge conflicts on compressed artifact\n" CBM_ARTIFACT_FILENAME - " binary merge=ours\n", - fp); - (void)fclose(fp); - } else { - (void)close(fd); - } + /* Order matters: attributes apply left to right and the `binary` + * macro expands to `-diff -merge -text`, so a trailing `binary` + * unsets `merge=ours` and the conflict prevention this file + * exists for never engages. The macro must come first. */ + (void)fputs("# Auto-generated by codebase-memory-mcp\n" + "# Prevent merge conflicts on compressed artifact\n" CBM_ARTIFACT_FILENAME + " binary merge=ours\n", + fp); + (void)fclose(fp); } /* Best-effort: configure merge driver */ diff --git a/tests/test_artifact.c b/tests/test_artifact.c index 6206463e3..8bbec658d 100644 --- a/tests/test_artifact.c +++ b/tests/test_artifact.c @@ -334,12 +334,30 @@ TEST(artifact_gitattributes_created) { PASS(); } +/* Read a small file through the UTF-8 path layer (wide on Windows), so a + * non-ASCII path is never misread by the test's own ANSI CRT call. Returns the + * byte count, or 0 when the file cannot be opened or is empty. */ +static size_t read_small_file_utf8(const char *path, char *buf, size_t cap) { + buf[0] = '\0'; + FILE *fp = cbm_fopen(path, "rb"); + if (!fp) { + return 0; + } + size_t rd = fread(buf, 1, cap - 1, fp); + (void)fclose(fp); + buf[rd] = '\0'; + return rd; +} + /* #1171: a repository (and a cache directory) under a non-ASCII path must * export, report the artifact present, and import — also onto an existing * cache db. On Windows the atomic writer used the ANSI CRT fopen + MoveFileExA * and the existence check the ANSI stat, so CJK characters outside the active - * code page made every step fail. The bytes are escaped UTF-8 for a CJK repo - * directory name and a CJK cache directory name (kept ASCII in source). */ + * code page made every step fail. The same export also writes the + * .gitattributes merge=ours protection, which ensure_gitattributes created + * with the ANSI CRT open(): on Windows it was silently missing. The bytes are + * escaped UTF-8 for a CJK repo directory name and a CJK cache directory name + * (kept ASCII in source). */ #define ART_CJK_REPO "\xe6\x8c\x81\xe4\xb9\x85\xe5\x8c\x96\xe5\xa4\x8d\xe7\x8e\xb0" #define ART_CJK_CACHE "\xe5\xaf\xbc\xe5\x85\xa5" TEST(artifact_roundtrip_non_ascii_paths) { @@ -356,6 +374,23 @@ TEST(artifact_roundtrip_non_ascii_paths) { ASSERT_TRUE(cbm_file_exists(zst)); ASSERT_TRUE(cbm_artifact_exists(repo)); + char ga[1024]; + snprintf(ga, sizeof(ga), "%s/.codebase-memory/.gitattributes", repo); + char ga_content[512]; + ASSERT_TRUE(read_small_file_utf8(ga, ga_content, sizeof(ga_content)) > 0); + ASSERT_NOT_NULL(strstr(ga_content, CBM_ARTIFACT_FILENAME " binary merge=ours")); + + /* Create-only-if-absent: a re-export never rewrites an existing + * .gitattributes the user may have edited. */ + static const char user_ga[] = "# user-owned\n"; + FILE *uga = cbm_fopen(ga, "wb"); + ASSERT_NOT_NULL(uga); + ASSERT_TRUE(fputs(user_ga, uga) >= 0); + ASSERT_EQ(fclose(uga), 0); + ASSERT_EQ(cbm_artifact_export(g_db, repo, "test-proj", CBM_ARTIFACT_FAST), 0); + ASSERT_EQ(read_small_file_utf8(ga, ga_content, sizeof(ga_content)), strlen(user_ga)); + ASSERT_STR_EQ(ga_content, user_ga); + char cache_dir[1024]; snprintf(cache_dir, sizeof(cache_dir), "%s/" ART_CJK_CACHE, g_tmpdir); ASSERT_TRUE(cbm_mkdir_p(cache_dir, 0755)); From 46a8f57f18e5e40b62c4d40d31062028fb336c48 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 00:42:05 +0200 Subject: [PATCH 07/34] fix(ui): index job carries the resolved root, not the request spelling POST /api/index resolves the root_path of the request into a 4 KB buffer, but the index job kept the original request string in a 1 KB field. The job, the worker log lines, the 202 reply and /api/index-status therefore reported the request spelling instead of the resolved root, and a path past 1023 bytes reached the daemon cut off. Store the resolved path in the job, give the field the capacity of the buffer it is copied from, and answer 400 for a path that would not fit instead of storing a shortened one. The status listing's escape buffer follows the new field size. The job's downstream, application_background_index, re-resolves what it receives, so a resolved input is a fixed point there and the daemon keys the job by the same string the route worked with. Tests drive the live route: a root outside a configured allowed root is answered 403 and no job starts; a root spelled through a ".." segment is stored and forwarded in resolved form; a root past 1 KB is either refused or forwarded whole. The last case reaches the accepted branch on Linux (PATH_MAX 4096); on macOS the path cannot resolve and the refusal branch is asserted. Signed-off-by: Martin Vogel --- src/ui/http_server.c | 36 +++++-- tests/test_httpd.c | 238 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 259 insertions(+), 15 deletions(-) diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 66f1affb8..7403897e7 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -160,6 +160,10 @@ static void handle_ui_config(cbm_http_conn_t *c, const cbm_http_req_t *req) { /* ── Server state ─────────────────────────────────────────────── */ #define MAX_INDEX_JOBS 4 +/* Capacity of the buffer POST /api/index resolves the root into, and of the + * job field that buffer is copied to: the job carries the resolved string + * whole, so the path handed to the daemon is the one the route checked. */ +#define INDEX_JOB_ROOT_CAP 4096 enum { HTTP_RUN_IDLE = 0, @@ -170,7 +174,7 @@ enum { typedef struct { cbm_http_server_t *server; - char root_path[1024]; + char root_path[INDEX_JOB_ROOT_CAP]; /* resolved form, see handle_index_start */ char project_name[256]; atomic_int status; /* 0=idle, 1=running, 2=done, 3=error */ char error_msg[256]; @@ -1182,7 +1186,7 @@ static void handle_index_start(cbm_http_server_t *server, cbm_http_conn_t *c, * it accepted roots the MCP path refused — an operator's boundary held on one * entry point and not the other. Canonicalize first: the policy is defined * over resolved paths, and a symlink would otherwise launder the verdict. */ - char canonical_root[4096]; + char canonical_root[INDEX_JOB_ROOT_CAP]; char boundary_err[1024]; if (!cbm_canonical_path(rpath, canonical_root, sizeof(canonical_root))) { yyjson_doc_free(doc); @@ -1198,6 +1202,15 @@ static void handle_index_start(cbm_http_server_t *server, cbm_http_conn_t *c, cbm_http_replyf(c, 403, g_cors_json, "{\"error\":\"%s\"}", escaped); return; } + /* The job carries exactly the string the checks above ran on. The field + * has the resolved buffer's capacity, so a resolved path always fits; the + * check keeps a resized field from ever handing the worker a cut-off path + * instead of the one that was checked. */ + if (strlen(canonical_root) >= INDEX_JOB_ROOT_CAP) { + yyjson_doc_free(doc); + cbm_http_replyf(c, 400, g_cors_json, "{\"error\":\"root_path too long\"}"); + return; + } /* Find free job slot */ int slot = -1; @@ -1226,7 +1239,7 @@ static void handle_index_start(cbm_http_server_t *server, cbm_http_conn_t *c, job->thread_started = false; } job->server = server; - snprintf(job->root_path, sizeof(job->root_path), "%s", rpath); + snprintf(job->root_path, sizeof(job->root_path), "%s", canonical_root); snprintf(job->project_name, sizeof(job->project_name), "%s", project_name); job->error_msg[0] = '\0'; atomic_store(&job->status, 1); @@ -1258,14 +1271,15 @@ static void handle_index_status(cbm_http_server_t *server, cbm_http_conn_t *c) { if (pos > 1) http_appendf(buf, sizeof(buf), &pos, ","); const char *ss = st == 1 ? "indexing" : st == 2 ? "done" : "error"; - /* root_path comes from POST /api/index and is up to 1023 bytes, so four - * occupied slots exceed this buffer. http_appendf pins pos to - * sizeof(buf) on truncation, so the separator and the close have to go - * through it as well rather than indexing raw. Both fields are free-form, - * so escape them — a quote in a path would otherwise end its JSON string - * early. */ - /* Escaping can double each byte: root_path is 1024, error_msg 256. */ - char esc_path[2048]; + /* root_path is the resolved root POST /api/index stored, up to + * INDEX_JOB_ROOT_CAP - 1 bytes, so one long path already exceeds this + * buffer. http_appendf pins pos to sizeof(buf) on truncation, so the + * separator and the close have to go through it as well rather than + * indexing raw. Both fields are free-form, so escape them — a quote in a + * path would otherwise end its JSON string early. */ + /* Escaping can double each byte: root_path is INDEX_JOB_ROOT_CAP, + * error_msg 256. */ + char esc_path[2 * INDEX_JOB_ROOT_CAP]; char esc_error[512]; cbm_json_escape(esc_path, (int)sizeof(esc_path), server->index_jobs[i].root_path); cbm_json_escape(esc_error, (int)sizeof(esc_error), diff --git a/tests/test_httpd.c b/tests/test_httpd.c index 322130c33..ee024a4d6 100644 --- a/tests/test_httpd.c +++ b/tests/test_httpd.c @@ -18,6 +18,7 @@ #include "../src/foundation/compat_thread.h" #include "../src/foundation/log.h" #include "../src/foundation/platform.h" +#include "../src/foundation/str_util.h" /* cbm_json_escape: expected index-status fragment */ #include "../src/cli/cli.h" #include "../src/daemon/host_internal.h" #include "../src/git/git_context.h" /* #798 follow-up: live-socket git-resolve repro */ @@ -29,6 +30,7 @@ #include #include +#include /* PATH_MAX: which platforms resolve a root past 1 KB */ #include #include #include @@ -627,7 +629,7 @@ static int th_server_start_with_watcher(th_server_t *ts, cbm_watcher_t *watcher) typedef struct { atomic_int calls; - char root_path[512]; + char root_path[4096]; /* the route resolves into a 4 KB buffer; record it whole */ char project_name[256]; } th_ui_index_executor_t; @@ -974,6 +976,10 @@ TEST(ui_server_process_kill_route_is_unavailable) { TEST(ui_server_routes_indexing_through_joinable_daemon_executor) { char *root = th_mktempdir("cbm_httpd_daemon_index"); ASSERT_NOT_NULL(root); + /* The executor receives the root in resolved form (the string the route + * checked), which on macOS differs from the /tmp spelling. */ + char resolved_root[4096]; + ASSERT_TRUE(cbm_canonical_path(root, resolved_root, sizeof(resolved_root))); th_ui_index_executor_t executor = {0}; atomic_init(&executor.calls, 0); th_server_t ts; @@ -998,7 +1004,7 @@ TEST(ui_server_routes_indexing_through_joinable_daemon_executor) { ASSERT_GT(response_length, 0); ASSERT_EQ(th_status(response), 202); ASSERT_TRUE(called); - ASSERT_STR_EQ(executor.root_path, root); + ASSERT_STR_EQ(executor.root_path, resolved_root); ASSERT_STR_EQ(executor.project_name, "ui-project"); th_cleanup(root); PASS(); @@ -1048,6 +1054,227 @@ TEST(ui_server_free_never_joins_active_index_worker) { PASS(); } +/* ── POST /api/index carries the resolved root ────────────────── */ + +/* The route resolves the request's root_path, runs the workspace check on the + * resolved form, and the job must then carry exactly that string: what was + * checked is what the daemon receives. These cases drive the real endpoint + * with the recording executor and read every place the stored root surfaces + * (the executor argument, the 202 reply, /api/index-status). */ + +typedef struct { + bool was_set; + char value[4096]; +} th_allowed_root_saved_t; + +/* Set (or, with NULL, clear) CBM_ALLOWED_ROOT and remember the previous value + * so the test can put it back before asserting — a failed assertion returns + * early, and a boundary left behind would leak into every later test. */ +static void th_allowed_root_set(th_allowed_root_saved_t *saved, const char *value) { + const char *current = getenv("CBM_ALLOWED_ROOT"); + saved->was_set = current != NULL; + snprintf(saved->value, sizeof(saved->value), "%s", current ? current : ""); + if (value) { + cbm_setenv("CBM_ALLOWED_ROOT", value, 1); + } else { + cbm_unsetenv("CBM_ALLOWED_ROOT"); + } +} + +static void th_allowed_root_restore(const th_allowed_root_saved_t *saved) { + if (saved->was_set) { + cbm_setenv("CBM_ALLOWED_ROOT", saved->value, 1); + } else { + cbm_unsetenv("CBM_ALLOWED_ROOT"); + } +} + +typedef struct { + th_ui_index_executor_t executor; + int status; /* HTTP status of the POST, -1 when no reply came */ + bool executor_called; /* only waited for after a 202 */ + char response[8192]; /* the raw POST reply */ + char index_status[16384]; /* GET /api/index-status, read after the POST */ +} th_ui_index_probe_t; + +/* Response body: everything after the header block, or "" when absent. */ +static const char *th_response_body(const char *response) { + const char *body = strstr(response, "\r\n\r\n"); + return body ? body + 4 : ""; +} + +/* Boot a server with the recording executor, POST one /api/index request, + * wait for the worker only when the route accepted the job, read + * /api/index-status and stop the server. Everything observed lands in the + * probe so the caller asserts after teardown. */ +static void th_ui_index_probe(th_ui_index_probe_t *probe, const char *root_path, + const char *project_name) { + memset(probe, 0, sizeof(*probe)); + atomic_init(&probe->executor.calls, 0); + probe->status = -1; + th_server_t ts; + ts.srv = cbm_http_server_new(0); + if (!ts.srv) { + return; + } + cbm_http_server_set_index_executor(ts.srv, th_ui_index_executor, &probe->executor); + if (th_server_thread_start(&ts.tid, ts.srv) != 0) { + (void)cbm_http_server_free(ts.srv); + return; + } + int port = cbm_http_server_port(ts.srv); + char body[8192]; + snprintf(body, sizeof(body), "{\"root_path\":\"%s\",\"project_name\":\"%s\"}", root_path, + project_name); + char request[8192 + 256]; + snprintf(request, sizeof(request), + "POST /api/index HTTP/1.1\r\nContent-Type: application/json\r\n" + "Content-Length: %zu\r\n\r\n%s", + strlen(body), body); + if (th_http(port, request, probe->response, sizeof(probe->response)) > 0) { + probe->status = th_status(probe->response); + } + if (probe->status == 202) { + probe->executor_called = th_wait_atomic_int(&probe->executor.calls, 1, 2000); + } + char status_request[128]; + snprintf(status_request, sizeof(status_request), + "GET /api/index-status HTTP/1.1\r\nHost: 127.0.0.1:%d\r\n\r\n", port); + (void)th_http(port, status_request, probe->index_status, sizeof(probe->index_status)); + th_server_stop(&ts); +} + +TEST(ui_server_index_start_refuses_root_outside_allowed) { + char *tmp = th_mktempdir("cbm_httpd_index_bound"); + ASSERT_NOT_NULL(tmp); + char base[256]; + snprintf(base, sizeof(base), "%s", tmp); + char allowed[512]; + char outside[512]; + snprintf(allowed, sizeof(allowed), "%s/allowed", base); + snprintf(outside, sizeof(outside), "%s/outside", base); + ASSERT_EQ(th_mkdir_p(allowed), 0); + ASSERT_EQ(th_mkdir_p(outside), 0); + /* The boundary is declared in resolved form, as allow-root records it. */ + char canonical_base[4096]; + ASSERT_TRUE(cbm_canonical_path(base, canonical_base, sizeof(canonical_base))); + char allowed_root[4096 + 16]; + snprintf(allowed_root, sizeof(allowed_root), "%s/allowed", canonical_base); + /* Spelled under the allowed root, resolving beside it: the verdict has to + * come from the resolved form. */ + char request_path[600]; + snprintf(request_path, sizeof(request_path), "%s/allowed/../outside", base); + + th_allowed_root_saved_t saved; + th_allowed_root_set(&saved, allowed_root); + th_ui_index_probe_t probe; + th_ui_index_probe(&probe, request_path, "bound"); + th_allowed_root_restore(&saved); + th_cleanup(base); + + ASSERT_EQ(probe.status, 403); + ASSERT_NOT_NULL(strstr(probe.response, "outside the allowed root")); + ASSERT_EQ(atomic_load(&probe.executor.calls), 0); + ASSERT_STR_EQ(th_response_body(probe.index_status), "[]"); + PASS(); +} + +TEST(ui_server_index_start_stores_resolved_root) { + char *tmp = th_mktempdir("cbm_httpd_index_root"); + ASSERT_NOT_NULL(tmp); + char base[256]; + snprintf(base, sizeof(base), "%s", tmp); + char inner[512]; + snprintf(inner, sizeof(inner), "%s/allowed/inner", base); + ASSERT_EQ(th_mkdir_p(inner), 0); + char canonical_base[4096]; + ASSERT_TRUE(cbm_canonical_path(base, canonical_base, sizeof(canonical_base))); + char allowed_root[4096 + 16]; + snprintf(allowed_root, sizeof(allowed_root), "%s/allowed", canonical_base); + /* The request names the allowed root through a ".." segment, so its + * spelling is not the resolved form (on macOS /tmp is itself a link to + * /private/tmp, so the prefix differs as well). */ + char request_path[600]; + snprintf(request_path, sizeof(request_path), "%s/allowed/inner/..", base); + ASSERT_TRUE(strcmp(request_path, allowed_root) != 0); + + th_allowed_root_saved_t saved; + th_allowed_root_set(&saved, allowed_root); + th_ui_index_probe_t probe; + th_ui_index_probe(&probe, request_path, "resolved"); + th_allowed_root_restore(&saved); + th_cleanup(base); + + ASSERT_EQ(probe.status, 202); + ASSERT_TRUE(probe.executor_called); + /* The daemon receives the string the check ran on, not the request + * spelling. */ + ASSERT_STR_EQ(probe.executor.root_path, allowed_root); + ASSERT_STR_EQ(probe.executor.project_name, "resolved"); + /* The 202 reply and the status listing report that same stored root. */ + char expected[8192 + 64]; + snprintf(expected, sizeof(expected), "\"path\":\"%s\"}", allowed_root); + ASSERT_NOT_NULL(strstr(probe.response, expected)); + char escaped[8192]; + cbm_json_escape(escaped, (int)sizeof(escaped), allowed_root); + snprintf(expected, sizeof(expected), "\"path\":\"%s\",\"error\"", escaped); + ASSERT_NOT_NULL(strstr(probe.index_status, expected)); + PASS(); +} + +TEST(ui_server_index_start_long_root_not_truncated) { + char *tmp = th_mktempdir("cbm_httpd_index_long"); + ASSERT_NOT_NULL(tmp); + char base[256]; + snprintf(base, sizeof(base), "%s", tmp); + /* A root well past 1 KB, the capacity the job used to hold, and well + * inside the 4 KB the route resolves; 200-byte components keep every + * component under NAME_MAX. */ + char component[201]; + memset(component, 'd', sizeof(component) - 1); + component[sizeof(component) - 1] = '\0'; + char deep[2048]; + int n = snprintf(deep, sizeof(deep), "%s", base); + while (n < 1400) { + n += snprintf(deep + n, sizeof(deep) - (size_t)n, "/%s", component); + } + bool created = th_mkdir_p(deep) == 0; + char canonical[4096]; + bool resolved = created && cbm_canonical_path(deep, canonical, sizeof(canonical)) != 0; + + /* No boundary here: the question is the length the job keeps. */ + th_allowed_root_saved_t saved; + th_allowed_root_set(&saved, NULL); + th_ui_index_probe_t probe; + th_ui_index_probe(&probe, deep, "long"); + th_allowed_root_restore(&saved); + th_cleanup(base); + + if (probe.status == 202) { + /* The platform resolved the path, so the worker has to receive it + * whole — the same string the route checked. */ + ASSERT_TRUE(resolved); + ASSERT_TRUE(probe.executor_called); + ASSERT_TRUE(strlen(probe.executor.root_path) > 1023); + ASSERT_STR_EQ(probe.executor.root_path, canonical); + } else { + /* The platform cannot resolve a path this long (macOS: PATH_MAX is + * 1024, so mkdir and stat refuse it before the route sees it); the + * route answers 400 and keeps nothing. */ + ASSERT_EQ(probe.status, 400); + ASSERT_EQ(atomic_load(&probe.executor.calls), 0); + ASSERT_STR_EQ(th_response_body(probe.index_status), "[]"); + } +#if defined(PATH_MAX) && PATH_MAX >= 2048 + /* Linux resolves it (PATH_MAX 4096), and that is the one platform where a + * cut-off could reach the worker; the accepted branch is the only honest + * outcome there, so the refusal branch may not stand in for it. */ + ASSERT_TRUE(created); + ASSERT_EQ(probe.status, 202); +#endif + PASS(); +} + TEST(ui_server_root_without_embedded_assets_is_not_found) { /* The frontend is linked into the image, so its availability is decided at * build time, not warmed at runtime: a binary built without --with-ui has @@ -2318,8 +2545,8 @@ TEST(ui_server_logs_escape_dense_no_overflow) { /* The index-status endpoint renders every active job into a fixed 2 KB stack * buffer. http_appendf clamps its own writes, but the separator and the closing - * bracket were raw indexes, so two jobs holding ~1 KB root paths (the field is - * 1024 bytes and the value comes straight from POST /api/index) pushed pos to + * bracket were raw indexes, so two jobs holding ~1 KB root paths (the field + * was 1024 bytes and now holds the 4 KB resolved root) pushed pos to * the clamp and the close then wrote past the buffer. Drive it through the real * endpoint with the index executor stubbed out, in a forked child so the * overflow surfaces as a killing signal. */ @@ -2458,6 +2685,9 @@ SUITE(httpd) { RUN_TEST(ui_server_process_kill_route_is_unavailable); RUN_TEST(ui_server_routes_indexing_through_joinable_daemon_executor); RUN_TEST(ui_server_free_never_joins_active_index_worker); + RUN_TEST(ui_server_index_start_refuses_root_outside_allowed); + RUN_TEST(ui_server_index_start_stores_resolved_root); + RUN_TEST(ui_server_index_start_long_root_not_truncated); RUN_TEST(ui_server_root_without_embedded_assets_is_not_found); RUN_TEST(ui_server_same_origin_request_is_allowed); RUN_TEST(ui_server_rejects_foreign_and_null_origins); From 7077695f2990818d300662cf9e3c4a3ceca601aa Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 02:23:12 +0200 Subject: [PATCH 08/34] fix(ui): size the index-status body for the roots the table holds GET /api/index-status rendered every job into a fixed 2 KB stack buffer with a clamp. Now that a job carries its resolved root in a 4 KB field, one long root alone exceeded that buffer and the listing came back as a cut JSON body with status 200. Build the body in a heap buffer sized from the table: per slot the entry framing plus the root and the error message at cbm_json_escape's widest expansion. The two strings are escaped straight into the body, the append helpers keep their clamp as a second guard, and the buffer goes through the memory core like the other variable-size responses. The 202 reply of POST /api/index printed the stored root unescaped, so a quote or backslash in the path ended its JSON string early. It is now escaped the way the listing escapes it. Tests: with every slot holding a root as long as the platform resolves (just under the 4 KB field on Linux, under PATH_MAX on macOS), the listing parses as JSON and carries each root whole, first with one job and then with the table full; a root containing a quote comes back parseable from the 202 reply, naming the resolved path. Signed-off-by: Martin Vogel --- src/ui/http_server.c | 81 +++++++++++----- tests/test_httpd.c | 224 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 25 deletions(-) diff --git a/src/ui/http_server.c b/src/ui/http_server.c index 7403897e7..f9f2ff0c6 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -28,6 +28,7 @@ #endif /* pipeline.h no longer needed — indexing runs as subprocess */ #include "foundation/log.h" +#include "foundation/mem_core.h" #include "foundation/platform.h" #include "foundation/secure_random.h" #include "foundation/sha256.h" @@ -164,6 +165,13 @@ static void handle_ui_config(cbm_http_conn_t *c, const cbm_http_req_t *req) { * job field that buffer is copied to: the job carries the resolved string * whole, so the path handed to the daemon is the one the route checked. */ #define INDEX_JOB_ROOT_CAP 4096 +#define INDEX_JOB_ERROR_CAP 256 +/* GET /api/index-status sizes its body from the table: per entry the framing + * below ({"slot":N,"status":"indexing","path":"","error":""} and a separator, + * well under 64 bytes) plus both strings at cbm_json_escape's widest expansion, + * six bytes for a control character (\u00XX). */ +#define INDEX_STATUS_ENTRY_FRAMING 64 +#define JSON_ESCAPE_MAX_EXPANSION 6 enum { HTTP_RUN_IDLE = 0, @@ -177,7 +185,7 @@ typedef struct { char root_path[INDEX_JOB_ROOT_CAP]; /* resolved form, see handle_index_start */ char project_name[256]; atomic_int status; /* 0=idle, 1=running, 2=done, 3=error */ - char error_msg[256]; + char error_msg[INDEX_JOB_ERROR_CAP]; cbm_thread_t thread; bool thread_started; atomic_int completed; @@ -1256,44 +1264,67 @@ static void handle_index_start(cbm_http_server_t *server, cbm_http_conn_t *c, } job->thread_started = true; + /* The reply names the stored root; escape it as the status listing does, + * or a quote or backslash in the path ends the JSON string early. */ + char esc_path[JSON_ESCAPE_MAX_EXPANSION * INDEX_JOB_ROOT_CAP]; + cbm_json_escape(esc_path, (int)sizeof(esc_path), job->root_path); cbm_http_replyf(c, 202, g_cors_json, "{\"status\":\"indexing\",\"slot\":%d,\"path\":\"%s\"}", - slot, job->root_path); + slot, esc_path); +} + +/* Append src JSON-escaped, under http_appendf's clamp contract: once *pos has + * reached bufsz nothing more is written. */ +static void http_append_json_escaped(char *buf, size_t bufsz, int *pos, const char *src) { + if (*pos < 0) { + return; + } + if ((size_t)*pos >= bufsz) { + *pos = (int)bufsz; + return; + } + *pos += cbm_json_escape(buf + *pos, (int)(bufsz - (size_t)*pos), src); } /* GET /api/index-status — returns status of all index jobs */ static void handle_index_status(cbm_http_server_t *server, cbm_http_conn_t *c) { - char buf[2048] = "["; - int pos = 1; + /* Size the body for a full table: every slot's framing plus its root and + * error message at the widest escape. A fixed 2 KB stack buffer used to + * clamp here, so one long resolved root (the field holds 4 KB) came back + * as a cut JSON body with status 200. The append helpers keep clamping, + * so this calculation is not the only thing keeping pos in range. */ + size_t buf_size = 2 + (size_t)MAX_INDEX_JOBS * (INDEX_STATUS_ENTRY_FRAMING + + JSON_ESCAPE_MAX_EXPANSION * + (INDEX_JOB_ROOT_CAP + INDEX_JOB_ERROR_CAP)); + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, buf_size); + if (!buf) { + cbm_http_replyf(c, 500, g_cors_json, "{\"error\":\"oom\"}"); + return; + } + int pos = 0; + http_appendf(buf, buf_size, &pos, "["); for (int i = 0; i < MAX_INDEX_JOBS; i++) { int st = atomic_load(&server->index_jobs[i].status); if (st == 0) continue; if (pos > 1) - http_appendf(buf, sizeof(buf), &pos, ","); + http_appendf(buf, buf_size, &pos, ","); const char *ss = st == 1 ? "indexing" : st == 2 ? "done" : "error"; - /* root_path is the resolved root POST /api/index stored, up to - * INDEX_JOB_ROOT_CAP - 1 bytes, so one long path already exceeds this - * buffer. http_appendf pins pos to sizeof(buf) on truncation, so the - * separator and the close have to go through it as well rather than - * indexing raw. Both fields are free-form, so escape them — a quote in a - * path would otherwise end its JSON string early. */ - /* Escaping can double each byte: root_path is INDEX_JOB_ROOT_CAP, - * error_msg 256. */ - char esc_path[2 * INDEX_JOB_ROOT_CAP]; - char esc_error[512]; - cbm_json_escape(esc_path, (int)sizeof(esc_path), server->index_jobs[i].root_path); - cbm_json_escape(esc_error, (int)sizeof(esc_error), - st == 3 ? server->index_jobs[i].error_msg : ""); - http_appendf(buf, sizeof(buf), &pos, - "{\"slot\":%d,\"status\":\"%s\",\"path\":\"%s\",\"error\":\"%s\"}", i, ss, - esc_path, esc_error); + /* Both fields are free-form, so escape them — a quote in a path would + * otherwise end its JSON string early. */ + http_appendf(buf, buf_size, &pos, "{\"slot\":%d,\"status\":\"%s\",\"path\":\"", i, ss); + http_append_json_escaped(buf, buf_size, &pos, server->index_jobs[i].root_path); + http_appendf(buf, buf_size, &pos, "\",\"error\":\""); + http_append_json_escaped(buf, buf_size, &pos, + st == 3 ? server->index_jobs[i].error_msg : ""); + http_appendf(buf, buf_size, &pos, "\"}"); } - http_appendf(buf, sizeof(buf), &pos, "]"); - if ((size_t)pos >= sizeof(buf)) { - pos = (int)sizeof(buf) - 1; + http_appendf(buf, buf_size, &pos, "]"); + if ((size_t)pos >= buf_size) { + pos = (int)buf_size - 1; } buf[pos] = '\0'; - cbm_http_replyf(c, 200, g_cors_json, "%s", buf); + cbm_http_reply_buf(c, 200, g_cors_json, buf, (size_t)pos); + cbm_free(CBM_MEM_CLASS_OTHER, buf); } static void unwatch_project(cbm_http_server_t *srv, const char *name) { diff --git a/tests/test_httpd.c b/tests/test_httpd.c index ee024a4d6..d020dc7da 100644 --- a/tests/test_httpd.c +++ b/tests/test_httpd.c @@ -29,6 +29,7 @@ #include "ui/http_server.h" #include #include +#include /* the index replies are asserted as parsed JSON */ #include /* PATH_MAX: which platforms resolve a root past 1 KB */ #include @@ -1275,6 +1276,60 @@ TEST(ui_server_index_start_long_root_not_truncated) { PASS(); } +/* The 202 reply names the stored root; a quote or backslash in it has to + * travel escaped or the reply is not JSON. */ +TEST(ui_server_index_start_reply_escapes_path) { + char *tmp = th_mktempdir("cbm_httpd_index_quote"); + ASSERT_NOT_NULL(tmp); + char base[256]; + snprintf(base, sizeof(base), "%s", tmp); + char root[512]; +#ifdef _WIN32 + /* A quote is not a legal file-name byte on Windows; the separators of the + * resolved path already need escaping there. */ + snprintf(root, sizeof(root), "%s\\plain", base); +#else + snprintf(root, sizeof(root), "%s/q\"uote", base); +#endif + ASSERT_EQ(th_mkdir_p(root), 0); + char canonical[4096]; + ASSERT_TRUE(cbm_canonical_path(root, canonical, sizeof(canonical))); + /* The request body is JSON too, so the root goes in escaped. */ + char request_path[1024]; + cbm_json_escape(request_path, (int)sizeof(request_path), root); + + th_ui_index_probe_t probe; + th_ui_index_probe(&probe, request_path, "quoted"); + th_cleanup(base); + + bool parsed = false; + int slot = -1; + char reply_path[4096] = ""; + const char *body = th_response_body(probe.response); + yyjson_doc *doc = yyjson_read(body, strlen(body), 0); + if (doc) { + yyjson_val *obj = yyjson_doc_get_root(doc); + yyjson_val *path = yyjson_obj_get(obj, "path"); + yyjson_val *slot_val = yyjson_obj_get(obj, "slot"); + parsed = yyjson_is_obj(obj); + if (yyjson_is_str(path)) { + snprintf(reply_path, sizeof(reply_path), "%s", yyjson_get_str(path)); + } + if (yyjson_is_int(slot_val)) { + slot = yyjson_get_int(slot_val); + } + yyjson_doc_free(doc); + } + + ASSERT_EQ(probe.status, 202); + ASSERT_TRUE(probe.executor_called); + ASSERT_STR_EQ(probe.executor.root_path, canonical); + ASSERT_TRUE(parsed); + ASSERT_STR_EQ(reply_path, canonical); + ASSERT_TRUE(slot >= 0); + PASS(); +} + TEST(ui_server_root_without_embedded_assets_is_not_found) { /* The frontend is linked into the image, so its availability is decided at * build time, not warmed at runtime: a binary built without --with-ui has @@ -2644,6 +2699,173 @@ TEST(ui_server_index_status_long_paths_no_overflow) { #endif } +/* Grow path in place to exactly target bytes with 'd' components, none + * longer than 200 so every component stays under NAME_MAX. */ +static void th_pad_path_to(char *path, size_t cap, int target) { + char fill[201]; + memset(fill, 'd', sizeof(fill) - 1); + fill[sizeof(fill) - 1] = '\0'; + int n = (int)strlen(path); + while (n < target && (size_t)n + 2 < cap) { + int room = target - n - 1; /* after the '/' */ + int len = room > 200 ? 200 : room; + if (len <= 0) { + break; + } + n += snprintf(path + n, cap - (size_t)n, "/%.*s", len, fill); + } +} + +/* Parse an index-status body and compare each entry's "path" with the root + * expected for its slot. Results go to out-params so the caller can tear + * down before asserting. */ +static void th_index_status_check(const char *body, int expected_count, char expected[][4096], + bool *parsed, int *count, bool *paths_whole) { + *parsed = false; + *count = 0; + *paths_whole = false; + yyjson_doc *doc = yyjson_read(body, strlen(body), 0); + if (!doc) { + return; + } + yyjson_val *arr = yyjson_doc_get_root(doc); + if (yyjson_is_arr(arr)) { + *parsed = true; + *count = (int)yyjson_arr_size(arr); + bool whole = *count == expected_count; + size_t idx; + size_t max_entries; + yyjson_val *entry; + yyjson_arr_foreach(arr, idx, max_entries, entry) { + yyjson_val *slot = yyjson_obj_get(entry, "slot"); + yyjson_val *path = yyjson_obj_get(entry, "path"); + int s = yyjson_is_int(slot) ? yyjson_get_int(slot) : -1; + whole = whole && s >= 0 && s < expected_count && yyjson_is_str(path) && + strcmp(yyjson_get_str(path), expected[s]) == 0; + } + *paths_whole = whole; + } + yyjson_doc_free(doc); +} + +/* The listing is sized for what the job table holds. Every slot takes a root + * as long as the platform resolves; the body has to parse as JSON and carry + * each root whole, first with one job and then with the table full. Roots + * enter the table the way the clamp test's do: real directories through the + * route, held open by the blocking executor so every slot stays occupied. */ +TEST(ui_server_index_status_long_roots_render_whole) { + char *tmp = th_mktempdir("cbm_status_whole"); + ASSERT_NOT_NULL(tmp); + char base[256]; + snprintf(base, sizeof(base), "%s", tmp); + /* Linux: just under the 4 KB field; the 4 KB request body still wraps it. */ + int target = 4096 - 160; +#if defined(_WIN32) + /* MAX_PATH: long roots are not characterised there; the listing is still + * checked for shape and content with every slot occupied. */ + target = 200; +#elif defined(PATH_MAX) && PATH_MAX < 4096 + /* macOS: the longest root the route can resolve; four still exceed 2 KB. */ + target = PATH_MAX - 96; +#endif + char deep[MAX_TEST_INDEX_JOBS][4096]; + char expected[MAX_TEST_INDEX_JOBS][4096]; + for (int j = 0; j < MAX_TEST_INDEX_JOBS; j++) { + snprintf(deep[j], sizeof(deep[j]), "%s/%d", base, j); + th_pad_path_to(deep[j], sizeof(deep[j]), target); + ASSERT_EQ((int)strlen(deep[j]), target); + ASSERT_EQ(th_mkdir_p(deep[j]), 0); + ASSERT_TRUE(cbm_canonical_path(deep[j], expected[j], sizeof(expected[j]))); + } + + th_ui_blocking_index_executor_t executor = {0}; + atomic_init(&executor.calls, 0); + atomic_init(&executor.release, 0); + th_server_t ts; + ts.srv = cbm_http_server_new(0); + ASSERT_NOT_NULL(ts.srv); + cbm_http_server_set_index_executor(ts.srv, th_ui_blocking_index_executor, &executor); + ASSERT_EQ(th_server_thread_start(&ts.tid, ts.srv), 0); + int port = cbm_http_server_port(ts.srv); + char status_request[128]; + snprintf(status_request, sizeof(status_request), + "GET /api/index-status HTTP/1.1\r\nHost: 127.0.0.1:%d\r\n\r\n", port); + + int accepted = 0; + bool started = true; + bool one_parsed = false; + bool one_whole = false; + bool all_parsed = false; + bool all_whole = false; + int one_count = 0; + int all_count = 0; + size_t one_body_len = 0; + size_t all_body_len = 0; + char listing[65536]; + for (int j = 0; j < MAX_TEST_INDEX_JOBS; j++) { + char body[4600]; + snprintf(body, sizeof(body), "{\"root_path\":\"%s\",\"project_name\":\"p%d\"}", deep[j], j); + char request[4900]; + snprintf(request, sizeof(request), + "POST /api/index HTTP/1.1\r\nContent-Type: application/json\r\n" + "Content-Length: %zu\r\n\r\n%s", + strlen(body), body); + char response[8192]; + int rn = th_http(port, request, response, sizeof(response)); + if (rn > 0 && th_status(response) == 202) { + accepted++; + } + /* Wait for the state the listing depends on: this job running. */ + started = started && th_wait_atomic_int(&executor.calls, j + 1, 5000); + if (j == 0 || j == MAX_TEST_INDEX_JOBS - 1) { + int n = th_http(port, status_request, listing, sizeof(listing)); + const char *status_body = n > 0 ? th_response_body(listing) : ""; + if (j == 0) { + one_body_len = strlen(status_body); + th_index_status_check(status_body, 1, expected, &one_parsed, &one_count, + &one_whole); + } else { + all_body_len = strlen(status_body); + th_index_status_check(status_body, MAX_TEST_INDEX_JOBS, expected, &all_parsed, + &all_count, &all_whole); + } + } + } + /* Release the workers and tear down; free refuses until each worker has + * flagged completion, the same way the active-worker test waits. */ + atomic_store(&executor.release, 1); + cbm_http_server_stop(ts.srv); + ASSERT_EQ(cbm_thread_join(&ts.tid), 0); + bool freed = false; + uint64_t deadline = cbm_now_ms() + 2000; + while (!freed && cbm_now_ms() < deadline) { + freed = cbm_http_server_free(ts.srv); + if (!freed) + cbm_usleep(1000); + } + th_cleanup(base); + + ASSERT_TRUE(freed); + ASSERT_EQ(accepted, MAX_TEST_INDEX_JOBS); + ASSERT_TRUE(started); + if (!one_parsed) { + char m[128]; + snprintf(m, sizeof(m), "one-job index-status body of %zu bytes is not JSON", one_body_len); + FAIL(m); + } + ASSERT_EQ(one_count, 1); + ASSERT_TRUE(one_whole); + if (!all_parsed) { + char m[128]; + snprintf(m, sizeof(m), "full-table index-status body of %zu bytes is not JSON", + all_body_len); + FAIL(m); + } + ASSERT_EQ(all_count, MAX_TEST_INDEX_JOBS); + ASSERT_TRUE(all_whole); + PASS(); +} + /* ── Suite ────────────────────────────────────────────────────── */ SUITE(httpd) { @@ -2651,6 +2873,7 @@ SUITE(httpd) { RUN_TEST(ui_server_browse_wide_dir_no_overflow); RUN_TEST(ui_server_logs_escape_dense_no_overflow); RUN_TEST(ui_server_index_status_long_paths_no_overflow); + RUN_TEST(ui_server_index_status_long_roots_render_whole); /* Parser / helpers */ RUN_TEST(httpd_parse_simple_get); RUN_TEST(httpd_parse_security_headers_and_rejects_duplicates); @@ -2688,6 +2911,7 @@ SUITE(httpd) { RUN_TEST(ui_server_index_start_refuses_root_outside_allowed); RUN_TEST(ui_server_index_start_stores_resolved_root); RUN_TEST(ui_server_index_start_long_root_not_truncated); + RUN_TEST(ui_server_index_start_reply_escapes_path); RUN_TEST(ui_server_root_without_embedded_assets_is_not_found); RUN_TEST(ui_server_same_origin_request_is_allowed); RUN_TEST(ui_server_rejects_foreign_and_null_origins); From f62d36d083e5a1e814431a573bb65fd9fd9bd37b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 05:03:35 +0200 Subject: [PATCH 09/34] test(cross-repo): free every matcher result the tests receive Since #2133 a cross-repo result owns heap memory: a wildcard run records each store it skips in result.skipped_projects, and the caller releases it with cbm_cross_repo_result_free(). Fifteen tests in test_cross_repo.c received a result and never released it. The one that seeds a pre-#768 store next to a wildcard target records a skip, so it leaked the skip array and the skipped project name: 32 bytes in 2 allocations, allocated in cr_record_skip. Every assertion still passed. The macOS leak lane (test-lsan-macos) was the run that reported it, because it builds with Homebrew LLVM and sets detect_leaks=1. The ordinary macOS runs, the local ones included, are built with Apple clang, where leak detection is off and cannot be turned on, so they stayed green. Each test now releases its result right after the fixture is torn down and before the first assertion, which is the shape the two tests that already released theirs use. The release therefore also runs when an assertion fails, and the scalar fields the assertions read stay valid after it. The repro_issue523 cases release their results the same way. They name their targets, so they record no skip and did not leak, but they hold a result and follow the same ownership rule. Signed-off-by: Martin Vogel --- tests/repro/repro_issue523.c | 7 +++++++ tests/test_cross_repo.c | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/repro/repro_issue523.c b/tests/repro/repro_issue523.c index 82d45b38d..14191e004 100644 --- a/tests/repro/repro_issue523.c +++ b/tests/repro/repro_issue523.c @@ -195,6 +195,7 @@ TEST(repro_issue523_crossrepo_http_calls_edge) { /* ── Cleanup ──────────────────────────────────────────────── */ rh_cleanup(&client, NULL); rh_cleanup(&server, NULL); + cbm_cross_repo_result_free(&result); /* * WHY RED: result.http_edges == 0 on current code. @@ -315,6 +316,7 @@ TEST(repro_issue523_scheme_stripped_url_match) { rh_cleanup(&client, NULL); rh_cleanup(&server, NULL); + cbm_cross_repo_result_free(&r); ASSERT_GTE(r.http_edges, 1); PASS(); } @@ -355,6 +357,7 @@ TEST(repro_issue523_template_fuzzy_match) { rh_cleanup(&client, NULL); rh_cleanup(&server, NULL); + cbm_cross_repo_result_free(&r); ASSERT_GTE(r.http_edges, 1); PASS(); } @@ -398,6 +401,7 @@ TEST(repro_issue523_reverse_direction_match) { rh_cleanup(&client, NULL); rh_cleanup(&server, NULL); + cbm_cross_repo_result_free(&r); ASSERT_GTE(r.http_edges, 1); ASSERT_GTE(client_cross, 1); PASS(); @@ -464,6 +468,9 @@ TEST(repro_issue523_idempotent_cross_edges) { rh_cleanup(&client, NULL); rh_cleanup(&server, NULL); + cbm_cross_repo_result_free(&r1); + cbm_cross_repo_result_free(&r2); + cbm_cross_repo_result_free(&r3); ASSERT_EQ(client_1, 1); ASSERT_EQ(server_1, 1); diff --git a/tests/test_cross_repo.c b/tests/test_cross_repo.c index 54c75cbd5..23d74fb5b 100644 --- a/tests/test_cross_repo.c +++ b/tests/test_cross_repo.c @@ -271,12 +271,14 @@ TEST(cross_repo_null_target_fails_without_dereference) { const char *targets[] = {NULL}; cbm_cross_repo_result_t result = cbm_cross_repo_match("null-target-source", targets, 1); rejected = result.failed; + cbm_cross_repo_result_free(&result); #else fflush(NULL); pid_t child = fork(); if (child == 0) { const char *targets[] = {NULL}; cbm_cross_repo_result_t result = cbm_cross_repo_match("null-target-source", targets, 1); + cbm_cross_repo_result_free(&result); _exit(result.failed ? 0 : 2); } int status = 0; @@ -308,6 +310,7 @@ TEST(cross_repo_wildcard_keeps_projects_containing_internal_tokens) { const char *targets[] = {"*"}; cbm_cross_repo_result_t result = cbm_cross_repo_match("wildcard-source", targets, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 4); @@ -367,6 +370,7 @@ TEST(cross_repo_wildcard_skips_pre768_store_issue2133) { cbm_cross_repo_result_t result = cbm_cross_repo_match("wild-src", targets, 1); int edges = cross_repo_count_edges(&fixture, "wild-src", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 1); @@ -394,6 +398,8 @@ TEST(cross_repo_named_pre768_target_fails_before_cleanup_issue2133) { cbm_cross_repo_result_t result = cbm_cross_repo_match("named-src", with_pre768, 2); int after = cross_repo_count_edges(&fixture, "named-src", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&initial); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(initial.failed); ASSERT_EQ(before, 1); @@ -579,6 +585,7 @@ TEST(cross_repo_scan_bound_counts_examined_rows_not_matches) { const char *target = "bounded-target"; cbm_cross_repo_result_t result = cbm_cross_repo_match("bounded-source", &target, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 1); @@ -608,6 +615,8 @@ TEST(cross_repo_propagates_delete_failure) { failed = cbm_cross_repo_match("delete-source", &target, 1); } cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&initial); + cbm_cross_repo_result_free(&failed); ASSERT_TRUE(trigger_created); ASSERT_TRUE(failed.failed); @@ -631,6 +640,7 @@ TEST(cross_repo_failed_bidirectional_insert_is_not_counted) { const char *target = "insert-target"; cbm_cross_repo_result_t result = cbm_cross_repo_match("insert-source", &target, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_TRUE(result.failed); ASSERT_EQ(result.http_edges, 0); @@ -683,6 +693,7 @@ TEST(cross_repo_cancel_mid_run_keeps_completed_target_and_stops_before_later_tar int later_target_edges = cross_repo_count_edges(&fixture, "cancel-target-c", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_EQ(hook.fired, 1); ASSERT_TRUE(result.cancelled); @@ -714,6 +725,8 @@ TEST(cross_repo_pre_cancel_preserves_existing_cross_edges) { cbm_cross_repo_match_cancellable("pre-cancel-source", &target, 1, &cancelled); int after = cross_repo_count_edges(&fixture, "pre-cancel-source", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&initial); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(initial.failed); ASSERT_TRUE(before > 0); @@ -747,6 +760,8 @@ TEST(cross_repo_self_only_target_fails_and_keeps_edges_issue1133) { cbm_cross_repo_result_t result = cbm_cross_repo_match("self-source", &self, 1); int after = cross_repo_count_edges(&fixture, "self-source", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&initial); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(initial.failed); ASSERT_TRUE(before > 0); @@ -770,6 +785,7 @@ TEST(cross_repo_wildcard_with_no_other_project_fails_issue1133) { const char *targets[] = {"*"}; cbm_cross_repo_result_t result = cbm_cross_repo_match("lonely-source", targets, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_TRUE(result.failed); ASSERT_TRUE(result.no_targets); @@ -820,6 +836,7 @@ TEST(cross_repo_accepts_project_with_missed_shadow_row_issue1609) { const char *target = "shadow-target"; cbm_cross_repo_result_t result = cbm_cross_repo_match("shadow-source", &target, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&result); ASSERT_FALSE(result.failed); ASSERT_EQ(result.projects_scanned, 1); @@ -847,6 +864,8 @@ TEST(cross_repo_caller_local_handler_blocks_cross_http_issue1459) { int mono_edges = cross_repo_count_edges(&fixture, "mono-1459", "CROSS_HTTP_CALLS"); int api_edges = cross_repo_count_edges(&fixture, "api-1459", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&fwd); + cbm_cross_repo_result_free(&rev); ASSERT_FALSE(fwd.failed); ASSERT_FALSE(rev.failed); @@ -880,6 +899,8 @@ TEST(cross_repo_methodless_call_to_local_get_route_blocks_cross_http_issue1459) int mono_edges = cross_repo_count_edges(&fixture, "mono2-1459", "CROSS_HTTP_CALLS"); int api_edges = cross_repo_count_edges(&fixture, "api3-1459", "CROSS_HTTP_CALLS"); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&fwd); + cbm_cross_repo_result_free(&rev); ASSERT_FALSE(fwd.failed); ASSERT_FALSE(rev.failed); @@ -908,6 +929,8 @@ TEST(cross_repo_caller_without_local_handler_keeps_cross_http_issue1459) { const char *consumer = "client-1459"; cbm_cross_repo_result_t rev = cbm_cross_repo_match("api2-1459", &consumer, 1); cross_repo_fixture_end(&fixture); + cbm_cross_repo_result_free(&fwd); + cbm_cross_repo_result_free(&rev); ASSERT_FALSE(fwd.failed); ASSERT_FALSE(rev.failed); From 44efa6281b4f6fa188ef095668e975346f66a619 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 06:37:36 +0200 Subject: [PATCH 10/34] fix(extract): C typedefs become nodes; macros and enumerators get their own QNs In C-family code several different entities competed for one node, and some had no node at all: - A typedef name was never a definition: `type_definition` has no `name` field, so the class path dropped it. With an anonymous `typedef struct/enum {...} X;` the fields and enumerators went with it. - A function behind an unknown leading macro (`API RetT name(...)`) was named after its return type, became a Method `RetT.name`, or was dropped, depending on the error-recovery shape. - Every bodyless `struct X` / `enum X` (a parameter, a field type, a forward declaration) minted a Class/Enum node. In one file, or in a same-stem .h/.c pair, such a reference displaced the real definition. - A `#define NAME` shared the QN of the function, type or enumerator called NAME, and the later line won: a namespace-rename macro replaced the typedef it renames, an `#else` stub macro replaced the function. - Enumerators were named `..`, although C and unscoped C++ enums put them in the enclosing scope. - A definition repeated in several `#if` branches kept one span; the others left no trace. - Braces split across `#if` branches lost whole functions, and a C keyword could surface as a function name (`if`). Extraction: - typedef names are definitions: `Type`, or `Class`/`Enum` with members for an anonymous `typedef struct/enum {...} X;`; an alias is dropped when the same file defines the tag of that name - macro-prefixed functions get their real name in all three recovery shapes, consistently for defs, call scopes, parameters and the C LSP - a bodyless tag is a reference and creates no node; a definition head left as loose ERROR tokens (`struct NAME {`) is recovered - a C keyword is never a function name - a first-branch projection (one branch per `#if` group, positions unchanged) is parsed when the raw parse lost structure wholesale, and only functions missing from the raw result are adopted from it - macro-wrapped enumerator lists yield one constant per slot Identity (semantic index version 4; an existing index is rebuilt once): - a C-preprocessor macro's QN is `.#macro` (C, C++, CUDA, GLSL, Objective-C, ISPC); name and label are unchanged. Resolvers prefer a definition and use the macro only when no definition of that name is visible (LSP target lookup, trace_path, get_code_snippet) - enumerators of C enums and of unscoped C++/Objective-C enums are `.`; `enum class` stays nested; `parent_class` names the enum - a node whose file defines its QN more than once under one label carries `variants`, the line spans of all those definitions (`#if` twins, per-platform macro redefinitions, C++ overloads) Full-text search indexes a macro's QN without the fence, so macro rows score as before. Measured on redis and curl against the parent commit: nodes 38,625 -> 40,484 and 28,180 -> 29,750; Type nodes 0 -> 573 and 0 -> 193; phantom Class nodes from references 202 and 821 removed; functions lost to a same-named macro 45 and 41 restored; curl functions with no node 11 -> 0; nodes of non-C-family files unchanged by QN. Of 174 audited Doxygen references in xxhash, 174 now reach the right entity (64 before). Index time is unchanged within noise (+2% pipeline time for 5% more nodes). Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 448 ++++++++++++++++++ internal/cbm/cbm.h | 21 + internal/cbm/extract_defs.c | 615 +++++++++++++++++++++++-- internal/cbm/helpers.c | 131 +++++- internal/cbm/helpers.h | 31 ++ internal/cbm/lsp/c_lsp.c | 28 +- internal/cbm/result_compact.c | 1 + src/graph_buffer/graph_buffer.c | 31 +- src/mcp/mcp.c | 24 + src/pipeline/lsp_resolve.h | 47 +- src/pipeline/pass_definitions.c | 40 +- src/pipeline/pass_parallel.c | 38 +- src/pipeline/pipeline_internal.h | 10 +- src/pipeline/registry.c | 4 +- src/store/store.c | 15 +- tests/test_extraction.c | 763 ++++++++++++++++++++++++++++++- tests/test_graph_buffer.c | 67 +++ tests/test_mcp.c | 119 +++++ tests/test_pipeline.c | 582 +++++++++++++++++++++++ tests/test_store_search.c | 62 +++ 20 files changed, 2997 insertions(+), 80 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 082bca1d6..ea6124a3a 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2098,6 +2098,444 @@ static const char *cbm_error_ranges_str(CBMArena *a, const cbm_error_regions_t * return buf; } +/* ── Same-file duplicate definitions: the `variants` property ────────────── + * + * The graph keeps one node per qualified name. When one file defines a QN more + * than once under one label -- both branches of an #if, a macro redefined per + * platform, C++ overloads -- the node shows the winner's span and the other + * definitions leave no trace. The member the graph keeps gets the group's + * spans as CBMDefinition.variants (format in cbm.h), so the surviving node + * carries them. + * + * C-preprocessor languages only (cbm_is_c_preprocessor_lang, the caller's + * gate): conditional compilation is what the property describes. A repeated + * QN elsewhere is a different matter -- a JSON file repeats one key thousands + * of times -- and gets no list. + * + * Computed here from this file's defs alone: a pure function of the file, + * hence deterministic, identical on the sequential and the parallel path, and + * owned by the file under incremental re-extraction. Two defs with the same + * span are one definition seen twice and count once; a group left with a + * single span gets nothing. */ + +typedef struct { + uint64_t key; /* hash of (QN, label) */ + uint32_t start; + uint32_t end; + int idx; /* index into result->defs */ +} cbm_variant_ref_t; + +static int cbm_variant_ref_cmp(const void *pa, const void *pb) { + const cbm_variant_ref_t *a = (const cbm_variant_ref_t *)pa; + const cbm_variant_ref_t *b = (const cbm_variant_ref_t *)pb; + if (a->key != b->key) { + return a->key < b->key ? -1 : 1; + } + if (a->start != b->start) { + return a->start < b->start ? -1 : 1; + } + if (a->end != b->end) { + return a->end < b->end ? -1 : 1; + } + return (a->idx > b->idx) - (a->idx < b->idx); +} + +/* FNV-1a over the QN, a separator and the label. */ +static uint64_t cbm_variant_key(const char *qn, const char *label) { + uint64_t h = 1469598103934665603ULL; + for (const char *p = qn; *p; p++) { + h = (h ^ (unsigned char)*p) * 1099511628211ULL; + } + h = (h ^ 0xffU) * 1099511628211ULL; + for (const char *p = label; *p; p++) { + h = (h ^ (unsigned char)*p) * 1099511628211ULL; + } + return h; +} + +/* refs[0..n) is one (QN, label) group in span order: give every member the + * list of the group's distinct spans. */ +static void cbm_variant_assign(CBMFileResult *result, const cbm_variant_ref_t *refs, int n) { + enum { VARIANT_ENTRY_MAX = 40 }; /* ,{"start":4294967295,"end":4294967295} */ + int distinct = 0; + for (int i = 0; i < n; i++) { + if (i == 0 || refs[i].start != refs[i - 1].start || refs[i].end != refs[i - 1].end) { + distinct++; + } + } + if (distinct < 2) { + return; + } + size_t cap = (size_t)distinct * VARIANT_ENTRY_MAX + 3; + char *json = (char *)cbm_arena_alloc(&result->arena, cap); + if (!json) { + return; + } + size_t pos = 0; + json[pos++] = '['; + for (int i = 0; i < n; i++) { + if (i > 0 && refs[i].start == refs[i - 1].start && refs[i].end == refs[i - 1].end) { + continue; + } + int w = snprintf(json + pos, cap - pos, "%s{\"start\":%u,\"end\":%u}", pos > 1 ? "," : "", + refs[i].start, refs[i].end); + if (w <= 0 || (size_t)w >= cap - pos) { + return; /* cannot happen: cap covers the longest entry */ + } + pos += (size_t)w; + } + json[pos++] = ']'; + json[pos] = '\0'; + /* Only the definition the graph keeps for this file carries the list: the + * one with the largest start line (cbm_gbuf_upsert_node's same-file rule; + * every member starting on that line, since arrival order breaks that + * tie). Giving it to all n members would serialize an n-entry list n + * times -- quadratic on a file that defines one name very often. */ + uint32_t last_start = refs[n - 1].start; + for (int i = n - 1; i >= 0 && refs[i].start == last_start; i--) { + result->defs.items[refs[i].idx].variants = json; + } +} + +static void cbm_mark_def_variants(CBMFileResult *result) { + enum { VARIANT_STACK_REFS = 128 }; + int n = result->defs.count; + if (n < 2) { + return; + } + cbm_variant_ref_t stack[VARIANT_STACK_REFS]; + cbm_variant_ref_t *refs = stack; + if (n > VARIANT_STACK_REFS) { + refs = (cbm_variant_ref_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)n * sizeof(*refs)); + if (!refs) { + return; + } + } + const CBMDefinition *defs = result->defs.items; + int m = 0; + for (int i = 0; i < n; i++) { + if (defs[i].qualified_name && defs[i].label) { + refs[m].key = cbm_variant_key(defs[i].qualified_name, defs[i].label); + refs[m].start = defs[i].start_line; + refs[m].end = defs[i].end_line; + refs[m].idx = i; + m++; + } + } + qsort(refs, (size_t)m, sizeof(*refs), cbm_variant_ref_cmp); + for (int i = 0; i < m;) { + int j = i + 1; + while (j < m && refs[j].key == refs[i].key) { + j++; + } + /* [i, j) shares one key. That is one (QN, label) group unless two + * keys collided, so split it by real equality, keeping span order. */ + int lo = i; + while (j - i >= 2 && lo < j) { + const CBMDefinition *lead = &defs[refs[lo].idx]; + int hi = lo + 1; + for (int k = lo + 1; k < j; k++) { + const CBMDefinition *d = &defs[refs[k].idx]; + if (strcmp(lead->qualified_name, d->qualified_name) == 0 && + strcmp(lead->label, d->label) == 0) { + cbm_variant_ref_t moved = refs[k]; + memmove(&refs[hi + 1], &refs[hi], (size_t)(k - hi) * sizeof(*refs)); + refs[hi++] = moved; + } + } + if (hi - lo >= 2) { + cbm_variant_assign(result, &refs[lo], hi - lo); + } + lo = hi; + } + i = j; + } + if (refs != stack) { + cbm_free(CBM_MEM_CLASS_EXTRACT, refs); + } +} + +/* ── First-branch projection rescue (C/C++/CUDA) ─────────────────────────── + * + * The raw parse sees every #if branch at once. When branches split a brace + * (`#ifndef X (a)) { #else (b)) { #endif`) or a directive sits inside an + * expression, the braces stop balancing and an ERROR region swallows whole + * functions: they get no node at all (curl ossl_connect_step2). The #961 + * rescue re-parses the preprocessed source, but with no build defines a + * whole-file guard (`#ifdef USE_OPENSSL`) leaves nothing to parse. The + * projection below needs no defines: every conditional group keeps exactly one + * branch (the first; for `#if 0` the #else), so the kept text is one + * consistent configuration the author wrote. Directive lines and dropped + * branches become spaces with newlines kept, so every byte keeps its line and + * column and a def found there needs no remapping. */ + +enum { CBM_PROJ_MAX_DEPTH = 64 }; + +typedef struct { + uint8_t branch; /* index of the branch being scanned (0 = the #if part) */ + uint8_t keep; /* branch that survives: 0, or 1 for `#if 0` */ +} cbm_proj_frame_t; + +/* Directive name after '#' and optional blanks, as [*name, *name + return). */ +static int cbm_proj_directive(const char *p, const char *e, const char **name) { + while (p < e && (*p == ' ' || *p == '\t')) { + p++; + } + if (p >= e || *p != '#') { + return 0; + } + p++; + while (p < e && (*p == ' ' || *p == '\t')) { + p++; + } + const char *n = p; + while (p < e && isalpha((unsigned char)*p)) { + p++; + } + *name = n; + return (int)(p - n); +} + +static bool cbm_proj_word_is(const char *n, int len, const char *w) { + return (size_t)len == strlen(w) && strncmp(n, w, (size_t)len) == 0; +} + +/* `#if 0` (the dead-code idiom): the condition is the single token 0. */ +static bool cbm_proj_if_zero(const char *after, const char *e) { + while (after < e && (*after == ' ' || *after == '\t')) { + after++; + } + if (after >= e || *after != '0') { + return false; + } + after++; + while (after < e && (*after == ' ' || *after == '\t' || *after == '\r')) { + after++; + } + return after >= e || (after + 1 < e && after[0] == '/' && (after[1] == '/' || after[1] == '*')); +} + +/* Track block-comment state across one line, skipping string and char + * literals, so a `#if` inside a block comment is not taken for a directive. */ +static bool cbm_proj_scan_comments(const char *p, const char *e, bool in_comment) { + while (p < e) { + if (in_comment) { + if (p + 1 < e && p[0] == '*' && p[1] == '/') { + in_comment = false; + p += 2; + continue; + } + p++; + continue; + } + if (*p == '"' || *p == '\'') { + char q = *p++; + while (p < e && *p != q) { + p += (*p == '\\' && p + 1 < e) ? 2 : 1; + } + p++; + continue; + } + if (p + 1 < e && p[0] == '/' && p[1] == '/') { + return false; + } + if (p + 1 < e && p[0] == '/' && p[1] == '*') { + in_comment = true; + p += 2; + continue; + } + p++; + } + return in_comment; +} + +static void cbm_proj_blank(char *p, const char *e) { + for (; p < e; p++) { + if (*p != '\n') { + *p = ' '; + } + } +} + +/* Heap copy of `src` with the first-branch projection applied, or NULL when + * the source has no conditional group (nothing to project), nests deeper than + * CBM_PROJ_MAX_DEPTH, or allocation fails. Caller frees with cbm_free. */ +static char *cbm_first_branch_projection(const char *src, int len) { + char *out = cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)len + 1); + if (!out) { + return NULL; + } + memcpy(out, src, (size_t)len); + out[len] = '\0'; + cbm_proj_frame_t frames[CBM_PROJ_MAX_DEPTH]; + int depth = 0; + bool any_group = false; + bool in_comment = false; + bool continuation = false; /* previous conditional-directive line ended in '\' */ + const char *end = out + len; + for (char *line = out; line < end;) { + char *nl = memchr(line, '\n', (size_t)(end - line)); + char *eol = nl ? nl : (char *)end; + bool active = true; + for (int i = 0; i < depth && active; i++) { + active = frames[i].branch == frames[i].keep; + } + bool blank = !active || continuation; + bool was_continuation = continuation; + continuation = false; + const char *name = NULL; + int nlen = (!in_comment && !was_continuation) ? cbm_proj_directive(line, eol, &name) : 0; + bool conditional = false; + if (nlen > 0) { + if (cbm_proj_word_is(name, nlen, "if") || cbm_proj_word_is(name, nlen, "ifdef") || + cbm_proj_word_is(name, nlen, "ifndef")) { + if (depth >= CBM_PROJ_MAX_DEPTH) { + cbm_free(CBM_MEM_CLASS_EXTRACT, out); + return NULL; + } + bool zero = + cbm_proj_word_is(name, nlen, "if") && cbm_proj_if_zero(name + nlen, eol); + frames[depth].branch = 0; + frames[depth].keep = zero ? 1 : 0; + depth++; + conditional = any_group = true; + } else if (cbm_proj_word_is(name, nlen, "elif") || + cbm_proj_word_is(name, nlen, "elifdef") || + cbm_proj_word_is(name, nlen, "elifndef") || + cbm_proj_word_is(name, nlen, "else")) { + if (depth > 0 && frames[depth - 1].branch < UINT8_MAX) { + frames[depth - 1].branch++; + } + conditional = true; + } else if (cbm_proj_word_is(name, nlen, "endif")) { + if (depth > 0) { + depth--; + } + conditional = true; + } + } + if (conditional) { + blank = true; + char *last = eol; + while (last > line && (last[-1] == '\r' || last[-1] == ' ' || last[-1] == '\t')) { + last--; + } + continuation = last > line && last[-1] == '\\'; + } else if (was_continuation) { + char *last = eol; + while (last > line && (last[-1] == '\r' || last[-1] == ' ' || last[-1] == '\t')) { + last--; + } + continuation = last > line && last[-1] == '\\'; + } + in_comment = cbm_proj_scan_comments(line, eol, in_comment); + if (blank) { + cbm_proj_blank(line, eol); + } + line = nl ? nl + 1 : (char *)end; + } + if (!any_group) { + cbm_free(CBM_MEM_CLASS_EXTRACT, out); + return NULL; + } + return out; +} + +/* Only a parse that lost structure wholesale is worth a second parse: the raw + * root is itself an ERROR, or one top-most error region spans this many lines. + * Small local errors (a macro the grammar cannot read) are the common case — + * about half the kernel's C files have one — and the projection cannot fix + * them; the gate keeps the extra parse to ~1% of kernel C bytes (sampled). */ +enum { CBM_PROJ_MIN_ERROR_LINES = 20 }; + +/* Adopt from the projection only what the raw pass lost: a Function/Method + * def that overlaps a raw ERROR region, whose QN no def already holds, whose + * name is on its first raw line, and whose span has definition syntax (the + * #961 gates). Everything else the projected walk produced is discarded. */ +static void cbm_rescue_defs_from_projection(CBMExtractCtx *raw_ctx, const TSLanguage *ts_lang) { + CBMFileResult *result = raw_ctx->result; + if (!ts_node_has_error(raw_ctx->root)) { + return; + } + cbm_error_regions_t regs = {{0}, {0}, 0, 0}; + bool broken = strcmp(ts_node_type(raw_ctx->root), "ERROR") == 0; + if (broken) { + /* The whole file failed: every line is error territory. */ + regs.starts[0] = 1; + regs.ends[0] = ts_node_end_point(raw_ctx->root).row + 1; + regs.count = 1; + } else { + cbm_collect_error_regions(raw_ctx->root, ®s, raw_ctx->source, raw_ctx->source_len); + for (int r = 0; r < regs.count && !broken; r++) { + broken = regs.ends[r] >= regs.starts[r] + CBM_PROJ_MIN_ERROR_LINES; + } + } + if (!broken) { + return; + } + char *projected = cbm_first_branch_projection(raw_ctx->source, raw_ctx->source_len); + if (!projected) { + return; + } + TSParser *parser = get_thread_parser(ts_lang, raw_ctx->language); + TSTree *tree = NULL; + if (parser) { + ts_parser_reset(parser); + CBMStringInput input = {projected, (uint32_t)raw_ctx->source_len}; + TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL}; + TSParseOptions opts = {0}; + tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); + } + CBMHashTable *held = tree ? cbm_ht_create(CBM_SZ_256) : NULL; + if (held) { + for (int i = 0; i < result->defs.count; i++) { + if (result->defs.items[i].qualified_name) { + cbm_ht_set(held, result->defs.items[i].qualified_name, &result->defs.items[i]); + } + } + CBMExtractCtx pctx = { + .arena = raw_ctx->arena, + .scratch = raw_ctx->scratch, + .result = result, + .source = projected, + .source_len = raw_ctx->source_len, + .language = raw_ctx->language, + .project = raw_ctx->project, + .rel_path = raw_ctx->rel_path, + .module_qn = result->module_qn, + .root = ts_tree_root_node(tree), + }; + int before = result->defs.count; + cbm_extract_definitions_without_module(&pctx); + int w = before; + for (int i = before; i < result->defs.count; i++) { + CBMDefinition *d = &result->defs.items[i]; + bool adopt = d->name && d->qualified_name && cbm_def_label_is_callable(d->label) && + !cbm_ht_has(held, d->qualified_name); + bool overlaps = false; + for (int r = 0; adopt && r < regs.count && !overlaps; r++) { + overlaps = d->start_line <= regs.ends[r] && d->end_line >= regs.starts[r]; + } + adopt = + adopt && overlaps && + cbm_line_contains(raw_ctx->source, raw_ctx->source_len, d->start_line, d->name) && + cbm_span_contains_callable_def(raw_ctx->source, raw_ctx->source_len, d->start_line, + d->end_line, d->name); + if (adopt) { + result->defs.items[w++] = *d; + cbm_ht_set(held, result->defs.items[w - 1].qualified_name, + &result->defs.items[w - 1]); + } + } + result->defs.count = w; + cbm_ht_free(held); + } + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, projected); +} + /* Public entry: run the extraction and journal completion. The DONE mark on * every ordinary return (including error/timeout results) tells the crash * supervisor this file did NOT kill the worker — only a file whose S has no @@ -2798,6 +3236,11 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C cbm_preprocessed_source_free(preprocessed); } atomic_fetch_add(&total_preprocess_ns, now_ns() - pp_start); + + /* Functions both the raw parse and the #961 preprocessed rescue lost + * (whole-file feature guards, directives inside expressions). Runs + * only when the raw tree has errors and the file has #if groups. */ + cbm_rescue_defs_from_projection(&ctx, ts_lang); } // Bottleneck call-context metrics. Each call is attributed to the INNERMOST @@ -2950,6 +3393,11 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C } } + /* The def list is final here (raw walk, preprocessed and projection rescues). */ + if (cbm_is_c_preprocessor_lang(language)) { + cbm_mark_def_variants(result); + } + result->imports_count = result->imports.count; // Accumulate profiling counters diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index c0bba7792..4bb02a22a 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -246,6 +246,16 @@ typedef struct { * HTTP_CALLS edge to base + path. Tail fields: zero-init stays valid. */ const char *http_client; const char *http_base_url; + /* Set when this FILE holds two or more definitions with this QN and label + * (`#if`/`#else` twins, a macro redefined per platform, overloads): the + * graph keeps one node per QN, and this lists every one of those + * definitions' line spans, the surviving one included, as a JSON array + * sorted by start line: [{"start":10,"end":14},{"start":20,"end":26}]. + * Carried by the definition the graph keeps for the file (the last by + * start line), which makes it the node's `variants` property. NULL on + * every other definition, and in every language outside + * cbm_is_c_preprocessor_lang. */ + const char *variants; } CBMDefinition; /* Argument captured from a call expression */ @@ -258,6 +268,17 @@ typedef struct { #define CBM_MAX_CALL_ARGS 8 +/* The qualified name of a C-preprocessor macro (`#define NAME ...` in C, C++, + * CUDA, Objective-C, GLSL, ISPC) is `.#macro`. C keeps macros + * apart from ordinary identifiers, so a macro and a function, type, variable or + * enumerator may legally share a name in one file; the fence gives the macro + * an identity of its own, and the plain `.` always belongs to + * the definition. `name` stays NAME and the label stays "Macro". + * + * Tie rule for anything that resolves a NAME or a plain QN to a node: the + * definition is the target, and the macro only when no definition is visible. */ +#define CBM_MACRO_QN_SUFFIX "#macro" + /* Byte offsets are meaningful only within the source buffer that produced * them. C/C++/CUDA run both raw and preprocessed extraction passes, and those * buffers can contain unrelated occurrences at the same numeric span. */ diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index b3b6e6acc..388166c09 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3,6 +3,7 @@ #include "helpers.h" #include "lang_specs.h" #include "foundation/constants.h" +#include "foundation/hash_table.h" #include "foundation/log.h" // cbm_log_error #include "foundation/mem_core.h" // cbm_realloc/cbm_free -- walk_defs stack #include "extract_node_stack.h" @@ -207,6 +208,9 @@ enum { RT_PAIR_SIZE = 2 }; // Forward declarations static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); +static void emit_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, + const char *kind, char *name); +static void extract_c_typedef(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, int depth_unused); static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec); static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); @@ -537,6 +541,13 @@ char *cbm_cpp_out_of_line_parent_class(CBMArena *a, TSNode node, const char *sou for (int depth = 0; depth < DECLARATOR_DEPTH_LIMIT && !ts_node_is_null(decl); depth++) { const char *dk = ts_node_type(decl); if (strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) { + if (cbm_c_qualifier_is_recovered(decl)) { + /* `API RetT name(...)` misread as `RetT::name` (MISSING "::"): + * RetT is the return type, not a class. Keep looking for a real + * qualifier on the name side. */ + decl = ts_node_child_by_field_name(decl, TS_FIELD("name")); + continue; + } qid = decl; break; } @@ -4961,6 +4972,11 @@ static TSNode find_c_params(TSNode func_node) { return params; } TSNode nested = ts_node_child_by_field_name(decl, TS_FIELD("declarator")); + /* `API RetT *name(...)` misread as `RetT::*name(...)` (MISSING "::"): + * the function declarator is on the qualifier's name side. */ + if (ts_node_is_null(nested) && cbm_c_qualifier_is_recovered(decl)) { + nested = ts_node_child_by_field_name(decl, TS_FIELD("name")); + } /* tree-sitter-cpp and tree-sitter-cuda do not assign the nested * function declarator a `declarator` field when a reference return * wraps it (`Item& operator[](int)`). That wrapper has one named child; @@ -5071,6 +5087,63 @@ static bool is_c_declarator_lang(CBMLanguage lang) { lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC; } +/* C-family `struct X` / `union X` / `enum X` / `class X` WITHOUT a body is a + * reference to the type — a forward declaration, a variable, parameter or field + * type, a sizeof operand, the aliased side of `typedef struct X Y;` — never its + * definition. Minting a def for it put a one-line Class/Enum node on the type's + * QN: when the definition shares that QN (same file, or a same-stem .h/.c pair, + * whose module QNs coincide), the later or smaller-path reference displaced the + * definition, and elsewhere it left a phantom type node in every file that + * merely mentions the type (`struct timeval` in redis-cli.c). */ +static bool is_c_tag_reference(CBMLanguage lang, TSNode node, const char *kind) { + if (!is_c_declarator_lang(lang)) { + return false; + } + if (strcmp(kind, "struct_specifier") != 0 && strcmp(kind, "union_specifier") != 0 && + strcmp(kind, "enum_specifier") != 0 && strcmp(kind, "class_specifier") != 0) { + return false; + } + return ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("body"))); +} + +/* Languages whose plain `enum` is UNSCOPED: its enumerators are names of the + * scope that holds the enum, not of the enum (C, C++ and its CUDA dialect, + * Objective-C). `XXH_OK` of `typedef enum { XXH_OK, XXH_ERROR } XXH_errorcode;` + * is written and looked up as plain XXH_OK, so its QN is `.XXH_OK`. */ +static bool c_enum_lang(CBMLanguage lang) { + return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || + lang == CBM_LANG_OBJC; +} + +/* Is a struct, union or class around an unscoped enum the scope of its + * enumerators? In C++ (and CUDA; every .h is parsed as C++) it is: the + * enumerator is `Brush::ROUND`. C and Objective-C have no struct scope for + * ordinary identifiers: an enum declared inside a struct puts its enumerators + * in the scope around the struct and the code names them unqualified, so the + * struct is no segment of their QN. */ +static bool c_enum_scope_is_class(CBMLanguage lang) { + return lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA; +} + +/* C++11 `enum class X` / `enum struct X`: the scoping keyword is an anonymous + * token child between `enum` and the body. Its enumerators stay `X::A`. */ +static bool c_enum_is_scoped(TSNode enum_node) { + TSNode body = ts_node_child_by_field_name(enum_node, TS_FIELD("body")); + uint32_t stop = ts_node_is_null(body) ? UINT32_MAX : ts_node_start_byte(body); + uint32_t nc = ts_node_child_count(enum_node); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(enum_node, i); + if (ts_node_start_byte(c) >= stop) { + break; + } + if (!ts_node_is_named(c) && + (strcmp(ts_node_type(c), "class") == 0 || strcmp(ts_node_type(c), "struct") == 0)) { + return true; + } + } + return false; +} + /* Render the canonical return type into `out`; returns how many qualifiers and * markers were added around the base type (0 = the base text alone is already * the whole type). The declarator walk is one strict child chain, so it is @@ -5957,6 +6030,26 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec if (extract_sql_ddl_class_def(ctx, node, kind)) { return; } + if (is_c_tag_reference(ctx->language, node, kind)) { + return; + } + /* A C/C++ typedef carries no `name` field (the alias names live in its + * declarators), so the generic path below always dropped it. */ + if (is_c_declarator_lang(ctx->language) && strcmp(kind, "type_definition") == 0) { + extract_c_typedef(ctx, node, spec); + return; + } + /* An anonymous C-family enum (`enum { A, B };`, also as a variable's or + * member's type) has no Enum def to mint, but its enumerators are names of + * the enclosing scope all the same. Inside a typedef, extract_c_typedef + * names the enum and emits them. */ + if (c_enum_lang(ctx->language) && strcmp(kind, "enum_specifier") == 0 && + ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("name")))) { + if (!doc_kind_is(doc_parent(ctx, node), "type_definition")) { + extract_enum_members(ctx, node, NULL); + } + return; + } TSNode name_node = ts_node_child_by_field_name(node, TS_FIELD("name")); // ObjC: class name is first identifier child @@ -6221,6 +6314,16 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec if (!name || !name[0]) { return; } + emit_class_def(ctx, node, spec, kind, name); +} + +/* Emit the class-like def for `node` under `name`, plus its members (enum + * members, methods, fields, class variables). Split from extract_class_def so a + * C `typedef struct { ... } Name;` can name its anonymous struct after the + * typedef (extract_c_typedef). */ +static void emit_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, + const char *kind, char *name) { + CBMArena *a = ctx->arena; // For nested classes, prefix with enclosing class QN (e.g., Outer.Inner). // Top-level classes use the language-aware module QN so Java/Go don't double @@ -6383,6 +6486,139 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec } } +/* The alias a C typedef declarator introduces: the type_identifier at the end + * of its pointer / array / function / parenthesized declarator chain + * (`typedef int (*cmp_fn)(const void *, const void *);` names cmp_fn). The C + * grammar lexes stdint-style names as primitive_type (`typedef unsigned + * uint32_t;` in a compat header), so that leaf counts as well. */ +static TSNode c_typedef_alias_node(TSNode decl) { + for (int depth = 0; depth < DECLARATOR_DEPTH_LIMIT && !ts_node_is_null(decl); depth++) { + const char *dk = ts_node_type(decl); + if (strcmp(dk, "type_identifier") == 0 || strcmp(dk, "primitive_type") == 0) { + return decl; + } + TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator")); + if (ts_node_is_null(inner) && ts_node_named_child_count(decl) > 0) { + inner = ts_node_named_child(decl, 0); + } + decl = inner; + } + TSNode null_node = {0}; + return null_node; +} + +/* C/C++ `typedef`: one def per alias name its declarators introduce. + * typedef struct { ... } Name; -> the anonymous struct IS Name: Class/Enum + * Name with its fields / enum members + * typedef struct Tag { ... } Tag; -> nothing extra: the struct def Tag (the + * walk reaches the specifier) is the entity + * typedef struct Tag Name; -> Type Name (alias; the struct is a + * typedef struct Tag Tag; reference, see is_c_tag_reference). An + * typedef unsigned int Name; identity alias of an incomplete struct is + * typedef int (*Name)(int); often the only declaration of an opaque + * handle type in the repo. + * The type_definition span and the doc comment above it belong to every alias. + * QN scheme unchanged: ., or . inside a + * C++ class body. drop_c_typedefs_shadowed_by_tags then removes an alias whose + * QN a struct/union/enum definition in the same file holds. */ +static void extract_c_typedef(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { + CBMArena *a = ctx->arena; + TSNode type = ts_node_child_by_field_name(node, TS_FIELD("type")); + const char *tk = ts_node_is_null(type) ? "" : ts_node_type(type); + bool tag_kind = strcmp(tk, "struct_specifier") == 0 || strcmp(tk, "union_specifier") == 0 || + strcmp(tk, "enum_specifier") == 0 || strcmp(tk, "class_specifier") == 0; + bool has_body = + tag_kind && !ts_node_is_null(ts_node_child_by_field_name(type, TS_FIELD("body"))); + TSNode tag_node = {0}; + if (tag_kind) { + tag_node = ts_node_child_by_field_name(type, TS_FIELD("name")); + } + /* Only a tag DEFINED here makes a same-named alias redundant. */ + const char *tag = + (ts_node_is_null(tag_node) || !has_body) ? NULL : cbm_node_text(a, tag_node, ctx->source); + bool anonymous_body = has_body && ts_node_is_null(tag_node); + + TSTreeCursor cursor = ts_tree_cursor_new(node); + bool more = ts_tree_cursor_goto_first_child(&cursor); + for (; more; more = ts_tree_cursor_goto_next_sibling(&cursor)) { + const char *field = ts_tree_cursor_current_field_name(&cursor); + if (!field || strcmp(field, "declarator") != 0) { + continue; + } + TSNode decl = ts_tree_cursor_current_node(&cursor); + TSNode alias_node = c_typedef_alias_node(decl); + char *name = ts_node_is_null(alias_node) ? NULL : cbm_node_text(a, alias_node, ctx->source); + if (!name || !name[0] || (tag && strcmp(tag, name) == 0)) { + continue; + } + if (anonymous_body && ts_node_eq(decl, alias_node)) { + /* its doc is the comment above the typedef (doc_anchor_c) */ + emit_class_def(ctx, type, spec, tk, name); + anonymous_body = false; /* a second plain name is an alias of the first */ + continue; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = name; + def.qualified_name = + ctx->enclosing_class_qn + ? cbm_arena_sprintf(a, "%s.%s", ctx->enclosing_class_qn, name) + : cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, name, ctx->language); + def.label = "Type"; + def.file_path = ctx->rel_path; + def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; + def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; + def.lines = (int)(def.end_line - def.start_line + TS_LINE_OFFSET); + def.is_exported = cbm_is_exported(name, ctx->language); + def.docstring = extract_docstring(ctx, node, name); + cbm_defs_push(&ctx->result->defs, a, def); + } + ts_tree_cursor_delete(&cursor); + /* No plain declarator named the anonymous enum (`typedef enum {A, B} *p;`): + * its enumerators are still names of the enclosing scope. */ + if (anonymous_body && c_enum_lang(ctx->language) && strcmp(tk, "enum_specifier") == 0) { + extract_enum_members(ctx, type, NULL); + } +} + +/* `typedef struct X X;` plus `struct X { ... };` in one file give the alias and + * the definition one QN, and the later line would win the graph upsert. The + * definition (fields, enum members, span) is the entity: drop the alias. Only + * defs from `first` on (this extraction pass) are considered, so the + * preprocessed rescue pass never filters the raw pass's defs. */ +static void drop_c_typedefs_shadowed_by_tags(CBMExtractCtx *ctx, int first) { + CBMDefArray *defs = &ctx->result->defs; + bool any_alias = false; + for (int i = first; i < defs->count && !any_alias; i++) { + any_alias = defs->items[i].label && strcmp(defs->items[i].label, "Type") == 0; + } + if (!any_alias) { + return; + } + CBMHashTable *tags = cbm_ht_create(CBM_SZ_64); + if (!tags) { + return; + } + for (int i = first; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->label && d->qualified_name && + (strcmp(d->label, "Class") == 0 || strcmp(d->label, "Enum") == 0)) { + cbm_ht_set(tags, d->qualified_name, (void *)d); + } + } + int w = first; + for (int i = first; i < defs->count; i++) { + const CBMDefinition *d = &defs->items[i]; + if (d->label && d->qualified_name && strcmp(d->label, "Type") == 0 && + cbm_ht_has(tags, d->qualified_name)) { + continue; + } + defs->items[w++] = defs->items[i]; + } + defs->count = w; + cbm_ht_free(tags); +} + // Find the body/members node inside a class node static TSNode find_class_body(TSNode class_node, CBMLanguage lang) { // Try field names first @@ -7184,40 +7420,144 @@ static bool is_enum_member_kind(const char *kind) { strcmp(kind, "enumerator") == 0; } +/* How an enum's members are named. + * + * The member QN is `.`, with one exception: a C-family + * UNSCOPED enum (c_enum_lang, not `enum class` / `enum struct`). Its enumerators + * live in the scope that holds the enum, so their QN is `.` + * and `class_qn` -- NULL for an anonymous enum -- is not a segment. That scope + * is the module, or in C++ the enclosing namespace or class; a C or + * Objective-C struct is none (c_enum_scope_is_class). Every C-family + * enumerator of a named enum carries `parent_class` = the enum's QN, which is + * what ties a flattened enumerator to its enum. */ +typedef struct { + const char *class_qn; /* the enum's QN; NULL for an anonymous enum */ + bool c_enum; /* a C-family enum_specifier */ + bool flat; /* the member QN leaves the enum's name out */ +} enum_member_scope_t; + +/* One enum member -> a Variable def. `doc_node` is the node whose leading + * comments document it: the member itself, or the ERROR that opens its slot + * in a macro-wrapped C list (see extract_c_enum_members). */ +static void push_enum_member(CBMExtractCtx *ctx, TSNode member, TSNode doc_node, + const enum_member_scope_t *scope) { + CBMArena *a = ctx->arena; + TSNode mname = ts_node_child_by_field_name(member, TS_FIELD("name")); + if (ts_node_is_null(mname)) { + mname = cbm_find_child_by_kind(member, "identifier"); + } + if (ts_node_is_null(mname)) { + return; + } + char *member_name = cbm_node_text(a, mname, ctx->source); + if (!member_name || !member_name[0]) { + return; + } + CBMDefinition mdef; + memset(&mdef, 0, sizeof(mdef)); + mdef.name = member_name; + if (!scope->flat) { + mdef.qualified_name = cbm_arena_sprintf(a, "%s.%s", scope->class_qn, member_name); + } else if (ctx->enclosing_class_qn && c_enum_scope_is_class(ctx->language)) { + mdef.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->enclosing_class_qn, member_name); + } else { + mdef.qualified_name = + cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, member_name, ctx->language); + } + if (scope->c_enum && scope->class_qn) { + mdef.parent_class = scope->class_qn; + } + mdef.label = "Variable"; + mdef.file_path = ctx->rel_path; + mdef.start_line = ts_node_start_point(member).row + TS_LINE_OFFSET; + mdef.end_line = ts_node_end_point(member).row + TS_LINE_OFFSET; + mdef.docstring = extract_member_docstring(ctx, doc_node); + cbm_defs_push(&ctx->result->defs, a, mdef); +} + +typedef struct { + int depth; /* parentheses an ERROR child opened and none closed yet */ + bool open; /* the current slot has no constant yet */ + bool has_head; /* an ERROR opened the current slot: `head` is that node */ + TSNode head; +} c_enum_slot_t; + +/* The parentheses and slot separators an ERROR child of the list carries. */ +static void c_enum_slot_scan_error(TSNode err, c_enum_slot_t *slot) { + if (slot->open && !slot->has_head) { + slot->head = err; + slot->has_head = true; + } + uint32_t n = ts_node_child_count(err); + for (uint32_t i = 0; i < n; i++) { + const char *k = ts_node_type(ts_node_child(err, i)); + if (strcmp(k, "(") == 0) { + slot->depth++; + } else if (strcmp(k, ")") == 0) { + if (slot->depth > 0) { + slot->depth--; + } + } else if (strcmp(k, ",") == 0 && slot->depth == 0) { + slot->open = true; + slot->has_head = false; + } + } +} + +/* A C-family enumerator_list under error recovery. A macro in the list -- + * CURLOPT(CURLOPT_URL, CURLOPTTYPE_STRINGPOINT, 2), + * CURLINFO_SPEED CURL_DEPRECATED(7.55.0, "...") = CURLINFO_DOUBLE + 9, + * -- is not enumerator grammar: the parser keeps every bare identifier it can + * as an `enumerator` and wraps the rest in ERROR nodes, so macro arguments and + * value operands come back as constants that do not exist (and, flattened, + * would take the plain QN of the macro they really are). The constant is the + * enumerator that OPENS a slot: the first one after `{` or after a `,` outside + * parentheses. A macro call that opens the slot hands that role to its first + * argument (the X-macro form), and the comment above the call documents it. + * A well-formed list has one enumerator per slot, so nothing changes there. */ +static void extract_c_enum_members(CBMExtractCtx *ctx, TSNode body, + const enum_member_scope_t *scope) { + c_enum_slot_t slot = {.depth = 0, .open = true, .has_head = false, .head = body}; + TSTreeCursor cur = ts_tree_cursor_new(body); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode child = ts_tree_cursor_current_node(&cur); + const char *ck = ts_node_type(child); + if (strcmp(ck, ",") == 0) { + if (slot.depth == 0) { + slot.open = true; + slot.has_head = false; + } + } else if (strcmp(ck, "ERROR") == 0) { + c_enum_slot_scan_error(child, &slot); + } else if (strcmp(ck, "enumerator") == 0 && slot.open) { + push_enum_member(ctx, child, slot.has_head ? slot.head : child, scope); + slot.open = false; + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); +} + /* Extract enum members as Variable nodes (C#, Java, TypeScript, C++). */ static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *class_qn) { - CBMArena *a = ctx->arena; TSNode body = find_class_body(node, ctx->language); if (ts_node_is_null(body)) { return; } + enum_member_scope_t scope = {.class_qn = class_qn, .c_enum = false, .flat = false}; + scope.c_enum = c_enum_lang(ctx->language) && strcmp(ts_node_type(node), "enum_specifier") == 0; + scope.flat = scope.c_enum && (!class_qn || !c_enum_is_scoped(node)); + if (scope.c_enum) { + extract_c_enum_members(ctx, body, &scope); + return; + } uint32_t mc = ts_node_named_child_count(body); for (uint32_t mi = 0; mi < mc; mi++) { TSNode member = ts_node_named_child(body, mi); - if (!is_enum_member_kind(ts_node_type(member))) { - continue; + if (is_enum_member_kind(ts_node_type(member))) { + push_enum_member(ctx, member, member, &scope); } - TSNode mname = ts_node_child_by_field_name(member, TS_FIELD("name")); - if (ts_node_is_null(mname)) { - mname = cbm_find_child_by_kind(member, "identifier"); - } - if (ts_node_is_null(mname)) { - continue; - } - char *member_name = cbm_node_text(a, mname, ctx->source); - if (!member_name || !member_name[0]) { - continue; - } - CBMDefinition mdef; - memset(&mdef, 0, sizeof(mdef)); - mdef.name = member_name; - mdef.qualified_name = cbm_arena_sprintf(a, "%s.%s", class_qn, member_name); - mdef.label = "Variable"; - mdef.file_path = ctx->rel_path; - mdef.start_line = ts_node_start_point(member).row + TS_LINE_OFFSET; - mdef.end_line = ts_node_end_point(member).row + TS_LINE_OFFSET; - mdef.docstring = extract_member_docstring(ctx, member); - cbm_defs_push(&ctx->result->defs, a, mdef); } } @@ -8982,6 +9322,202 @@ static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enc free(kids); } +/* 1-based line of the `}` that closes the `{` at `open`, or 0 when it does not + * close before `limit`. Counts only the first branch of every #if group (the + * first-branch projection rule of the C rescue in cbm.c): `#ifdef X struct { + * #else union { #endif` opens one brace, not two. Comments, string and char + * literals are skipped; dropped branches are ignored wholesale. */ +enum { C_BRACE_PP_DEPTH = 64 }; + +static bool c_brace_directive_is(const char *p, const char *e, const char *w) { + size_t n = strlen(w); + return (size_t)(e - p) >= n && strncmp(p, w, n) == 0 && + ((size_t)(e - p) == n || !isalpha((unsigned char)p[n])); +} + +static uint32_t c_matching_brace_line(const char *src, uint32_t open, uint32_t limit, + uint32_t open_row) { + uint8_t branch[C_BRACE_PP_DEPTH]; + uint8_t keep[C_BRACE_PP_DEPTH]; + int pp = 0; + int depth = 0; + uint32_t row = open_row; + bool line_start = false; + for (uint32_t i = open; i < limit; i++) { + char c = src[i]; + if (c == '\n') { + row++; + line_start = true; + continue; + } + if (line_start && (c == ' ' || c == '\t')) { + continue; + } + if (line_start && c == '#') { + const char *p = src + i + 1; + const char *e = src + limit; + while (p < e && (*p == ' ' || *p == '\t')) { + p++; + } + if (c_brace_directive_is(p, e, "if") || c_brace_directive_is(p, e, "ifdef") || + c_brace_directive_is(p, e, "ifndef")) { + if (pp >= C_BRACE_PP_DEPTH) { + return 0; + } + const char *q = p + 2; + while (q < e && isalpha((unsigned char)*q)) { + q++; + } + while (q < e && (*q == ' ' || *q == '\t')) { + q++; + } + bool zero = c_brace_directive_is(p, e, "if") && q < e && *q == '0' && + (q + 1 >= e || !isalnum((unsigned char)q[1])); + branch[pp] = 0; + keep[pp] = zero ? 1 : 0; + pp++; + } else if (c_brace_directive_is(p, e, "elif") || c_brace_directive_is(p, e, "else") || + c_brace_directive_is(p, e, "elifdef") || + c_brace_directive_is(p, e, "elifndef")) { + if (pp > 0) { + branch[pp - 1] = branch[pp - 1] < UINT8_MAX ? branch[pp - 1] + 1 : UINT8_MAX; + } else { + /* the next branch of a group opened before the brace: drop + * everything up to that group's #endif */ + branch[0] = 1; + keep[0] = 0; + pp = 1; + } + } else if (c_brace_directive_is(p, e, "endif") && pp > 0) { + pp--; + } + /* skip the directive's logical line, backslash continuations included */ + for (;;) { + while (i + 1 < limit && src[i + 1] != '\n') { + i++; + } + if (i + 1 < limit && src[i] == '\\') { + i++; /* onto the continued line's newline */ + row++; + continue; + } + break; + } + continue; + } + line_start = false; + bool active = true; + for (int k = 0; k < pp && active; k++) { + active = branch[k] == keep[k]; + } + if (!active) { + continue; + } + if (c == '/' && i + 1 < limit && src[i + 1] == '*') { + for (i += 2; i + 1 < limit && !(src[i] == '*' && src[i + 1] == '/'); i++) { + row += src[i] == '\n'; + } + i++; + continue; + } + if (c == '/' && i + 1 < limit && src[i + 1] == '/') { + while (i + 1 < limit && src[i + 1] != '\n') { + i++; + } + continue; + } + if (c == '"' || c == '\'') { + for (i++; i < limit && src[i] != c && src[i] != '\n'; i++) { + if (src[i] == '\\' && i + 1 < limit) { + i++; + row += src[i] == '\n'; + } + } + if (i < limit && src[i] == '\n') { + i--; /* unterminated literal: let the loop count the newline */ + } + continue; + } + if (c == '{') { + depth++; + } else if (c == '}' && --depth == 0) { + return row + TS_LINE_OFFSET; + } + } + return 0; +} + +/* tree-sitter can give up on a whole region and leave a definition head as loose + * ERROR tokens: `struct task_struct {` in the kernel's include-guarded sched.h is + * never recovered as a struct_specifier, so the type had no definition node (its + * only nodes were references, now dropped by is_c_tag_reference). The tokens + * `struct|union|class|enum NAME {` can only start a definition of NAME; recover it + * with its span up to the matching brace (members stay with the ERROR region). */ +static void recover_c_error_tag_heads(CBMExtractCtx *ctx, TSNode error_node) { + CBMArena *a = ctx->arena; + /* One linear cursor pass with a three-token window: a file-level ERROR can + * hold every top-level token of the file as a flat sibling, where indexed + * child access is quadratic (see wd_collect_children). */ + TSTreeCursor cursor = ts_tree_cursor_new(error_node); + TSNode win[3]; + memset(win, 0, sizeof(win)); + uint32_t seen = 0; + /* Once one head never closes, later heads in this region will not either: + * do not rescan to the region end for each of them. */ + bool unclosed = false; + bool more = ts_tree_cursor_goto_first_child(&cursor); + for (; more; more = ts_tree_cursor_goto_next_sibling(&cursor)) { + win[0] = win[1]; + win[1] = win[2]; + win[2] = ts_tree_cursor_current_node(&cursor); + if (++seen < 3) { + continue; + } + TSNode kw = win[0]; + TSNode name_node = win[1]; + TSNode brace = win[2]; + if (ts_node_is_named(kw)) { + continue; + } + const char *k = ts_node_type(kw); + bool is_enum = strcmp(k, "enum") == 0; + if (!is_enum && strcmp(k, "struct") != 0 && strcmp(k, "union") != 0 && + strcmp(k, "class") != 0) { + continue; + } + if (strcmp(ts_node_type(name_node), "type_identifier") != 0 || ts_node_is_named(brace) || + strcmp(ts_node_type(brace), "{") != 0) { + continue; + } + char *name = cbm_node_text(a, name_node, ctx->source); + if (!name || !name[0]) { + continue; + } + uint32_t end_line = unclosed ? 0 + : c_matching_brace_line(ctx->source, ts_node_start_byte(brace), + ts_node_end_byte(error_node), + ts_node_start_point(brace).row); + unclosed = end_line == 0; + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = name; + def.qualified_name = + ctx->enclosing_class_qn + ? cbm_arena_sprintf(a, "%s.%s", ctx->enclosing_class_qn, name) + : cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, name, ctx->language); + def.label = is_enum ? "Enum" : "Class"; + def.file_path = ctx->rel_path; + def.start_line = ts_node_start_point(kw).row + TS_LINE_OFFSET; + def.end_line = end_line ? end_line : ts_node_end_point(error_node).row + TS_LINE_OFFSET; + def.lines = (int)(def.end_line - def.start_line + TS_LINE_OFFSET); + def.is_exported = cbm_is_exported(name, ctx->language); + /* The doc comment sits before the ERROR node when the head opens it. */ + def.docstring = extract_docstring(ctx, seen == 3 ? error_node : kw, name); + cbm_defs_push(&ctx->result->defs, a, def); + } + ts_tree_cursor_delete(&cursor); +} + // Push nested class nodes from a class body container onto the defs stack. // Iteratively walks into wrapper nodes (field_declaration, template_declaration). static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s, @@ -9328,17 +9864,16 @@ static void extract_janet_def(CBMExtractCtx *ctx, TSNode node) { cbm_defs_push(&ctx->result->defs, a, def); } -// Languages that use the C preprocessor and therefore have #define macros. -static bool is_c_preprocessor_lang(CBMLanguage lang) { - return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || - lang == CBM_LANG_GLSL || lang == CBM_LANG_OBJC || lang == CBM_LANG_ISPC; -} - -// C/C++ preprocessor macros become Macro nodes (#375): +// C/C++ preprocessor macros become Macro nodes (#375), in the languages of +// cbm_is_c_preprocessor_lang: // #define SIMPLE 1 -> preproc_def // #define FN(x) (2 * (x)) -> preproc_function_def // The name is the `name` field; a function-like macro's parameter list is kept // as the signature. The macro body (a preproc_arg) is not descended into. +// The QN is `.#macro` (CBM_MACRO_QN_SUFFIX): macros have a +// namespace of their own, so a rename macro (`#define XXH32 XXH_NAME2(...)`) or +// an #else stand-in (`#define match(c, m) TRUE`) never competes with the +// function, type or enumerator of the same name for one node. static void extract_c_macro_def(CBMExtractCtx *ctx, TSNode node) { CBMArena *a = ctx->arena; TSNode name_node = ts_node_child_by_field_name(node, TS_FIELD("name")); @@ -9350,10 +9885,15 @@ static void extract_c_macro_def(CBMExtractCtx *ctx, TSNode node) { return; } + const char *plain_qn = cbm_fqn_compute(a, ctx->project, ctx->rel_path, name); + if (!plain_qn) { + return; + } + CBMDefinition def; memset(&def, 0, sizeof(def)); def.name = name; - def.qualified_name = cbm_fqn_compute(a, ctx->project, ctx->rel_path, name); + def.qualified_name = cbm_arena_sprintf(a, "%s" CBM_MACRO_QN_SUFFIX, plain_qn); def.label = "Macro"; def.file_path = ctx->rel_path; def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; @@ -9628,13 +10168,18 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec, if (ctx->language == CBM_LANG_KOTLIN && strcmp(kind, "ERROR") == 0) { recover_kotlin_error_classes(ctx, node); } + /* C/C++: a definition head left as loose ERROR tokens (additive, as + * above — descent below still visits the region's parsed subtrees). */ + if (is_c_declarator_lang(ctx->language) && strcmp(kind, "ERROR") == 0) { + recover_c_error_tag_heads(ctx, node); + } if (ctx->language == CBM_LANG_ELIXIR && strcmp(kind, "call") == 0) { extract_elixir_call(ctx, node, spec); continue; } - if (is_c_preprocessor_lang(ctx->language) && + if (cbm_is_c_preprocessor_lang(ctx->language) && (strcmp(kind, "preproc_def") == 0 || strcmp(kind, "preproc_function_def") == 0)) { // Gated to full/advanced index modes — macros dominate extraction on // macro-dense codebases (e.g. the Linux kernel). See #375. @@ -9786,7 +10331,11 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) { } // Walk AST for function/class definitions + int first = ctx->result->defs.count; walk_defs(ctx, ctx->root, spec, 0); + if (is_c_declarator_lang(ctx->language)) { + drop_c_typedefs_shadowed_by_tags(ctx, first); + } // Extract module-level variables extract_variables(ctx, ctx->root, spec); diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 4c4d340f0..03cc8b685 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -988,16 +988,109 @@ static TSNode resolve_qualified_name(TSNode decl) { return null_node; } +bool cbm_c_qualifier_is_recovered(TSNode qid) { + if (ts_node_is_null(qid)) { + return false; + } + uint32_t nc = ts_node_child_count(qid); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(qid, i); + if (!ts_node_is_named(c) && strcmp(ts_node_type(c), "::") == 0) { + return ts_node_is_missing(c); + } + } + return false; +} + +/* A function DEFINITION's parameter list: empty, `(void)`, or every parameter + * named (variadic `...` allowed). A function-like macro invocation that only + * looks like a declarator — `TEST_BEGIN(test_name)`, whose one "parameter" is + * an untyped identifier read as a type — fails this, so its macro name is never + * mistaken for the defined function's name. */ +static bool c_params_name_every_parameter(TSNode params) { + enum { MAX_PARAMS_CHECKED = 64 }; /* indexed child access: keep it small */ + uint32_t nc = ts_node_named_child_count(params); + if (nc > MAX_PARAMS_CHECKED) { + return false; + } + for (uint32_t i = 0; i < nc; i++) { + TSNode p = ts_node_named_child(params, i); + const char *pk = ts_node_type(p); + if (strcmp(pk, "comment") == 0 || strcmp(pk, "variadic_parameter") == 0) { + continue; + } + if (!ts_node_is_null(ts_node_child_by_field_name(p, TS_FIELD("declarator")))) { + continue; + } + /* `(void)`: the one parameter is an unnamed primitive type. */ + TSNode type = ts_node_child_by_field_name(p, TS_FIELD("type")); + if (nc != 1 || ts_node_is_null(type) || strcmp(ts_node_type(type), "primitive_type") != 0) { + return false; + } + } + return true; +} + +TSNode cbm_c_recovered_func_name(TSNode func_declarator) { + TSNode null_node = {0}; + if (ts_node_is_null(func_declarator) || + strcmp(ts_node_type(func_declarator), "function_declarator") != 0) { + return null_node; + } + TSNode params = ts_node_child_by_field_name(func_declarator, TS_FIELD("parameters")); + if (ts_node_is_null(params) || !c_params_name_every_parameter(params)) { + return null_node; + } + TSNode err = ts_node_prev_sibling(params); + if (ts_node_is_null(err) || strcmp(ts_node_type(err), "ERROR") != 0) { + return null_node; + } + /* Only the macro-prefix shape: the ERROR holds nothing but bare identifiers + * (the real name, possibly after further attribute macros). Any other token + * means the region is not a declaration this rule can vouch for. */ + enum { MAX_PREFIX_TOKENS = 4 }; + uint32_t nc = ts_node_child_count(err); + if (nc == 0 || nc > MAX_PREFIX_TOKENS) { + return null_node; + } + TSNode last = null_node; + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(err, i); + const char *ck = ts_node_type(c); + if (strcmp(ck, "identifier") != 0 && strcmp(ck, "type_identifier") != 0) { + return null_node; + } + last = c; + } + return last; +} + // Resolve function name from C/C++/CUDA/GLSL declarator chain. Shared canonical // implementation — see the header for the full rationale (#438). TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) { TSNode decl = ts_node_child_by_field_name(func_node, TS_FIELD("declarator")); + /* Set once a recovered `RetT::name` qualifier was stepped through: on that + * path the C++ grammar names the function with a type_identifier. */ + bool recovered = false; for (int depth = 0; depth < CBM_DECLARATOR_DEPTH_LIMIT && !ts_node_is_null(decl); depth++) { const char *dk = ts_node_type(decl); - if (is_c_terminal_name(dk)) { + if (is_c_terminal_name(dk) || (recovered && strcmp(dk, "type_identifier") == 0)) { return decl; } + if (strcmp(dk, "function_declarator") == 0) { + TSNode real = cbm_c_recovered_func_name(decl); + if (!ts_node_is_null(real)) { + return real; + } + } if (strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) { + if (cbm_c_qualifier_is_recovered(decl)) { + /* `API RetT name(...)`: the "scope" is the return type, the name + * side carries the real declarator chain. */ + decl = ts_node_child_by_field_name(decl, TS_FIELD("name")); + recovered = true; + continue; + } return resolve_qualified_name(decl); } TSNode inner = ts_node_child_by_field_name(decl, TS_FIELD("declarator")); @@ -1013,6 +1106,36 @@ TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) { return null_node; } +/* The C-declarator grammars: same set resolve_func_name_c_family routes through + * cbm_resolve_c_declarator_name_node. */ +static bool c_declarator_lang(CBMLanguage lang) { + return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || + lang == CBM_LANG_GLSL || lang == CBM_LANG_HLSL || lang == CBM_LANG_ISPC || + lang == CBM_LANG_SLANG || lang == CBM_LANG_OBJC; +} + +bool cbm_c_reserved_func_name(const char *name) { + /* Statement/operator keywords: error recovery over preprocessor-split code + * reads `else if (a) (b) {` as a definition `else if(...) {...}`. */ + static const char *const kw[] = {"if", "else", "for", "while", "do", + "switch", "case", "default", "return", "break", + "continue", "goto", "sizeof", "typedef", NULL}; + if (!name) { + return false; + } + for (const char *const *k = kw; *k; k++) { + if (strcmp(name, *k) == 0) { + return true; + } + } + return false; +} + +bool cbm_is_c_preprocessor_lang(CBMLanguage lang) { + return lang == CBM_LANG_C || lang == CBM_LANG_CPP || lang == CBM_LANG_CUDA || + lang == CBM_LANG_GLSL || lang == CBM_LANG_OBJC || lang == CBM_LANG_ISPC; +} + // Convert a resolved function/method name node to its name string. Most nodes // map directly to their text, but a C++ conversion-operator's `operator_cast` // node spans the full "operator bool() const" — this grammar folds the parameter @@ -1023,6 +1146,12 @@ TSNode cbm_resolve_c_declarator_name_node(TSNode func_node) { // `if (obj)`) misses. char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source, CBMLanguage lang) { char *text = cbm_node_text(a, name_node, source); + /* A C keyword is never a function name: the definition is an error-recovery + * artifact. No name means no def and no call scope (calls inside fall back + * to the enclosing scope), the same as any unnamed definition. */ + if (text && c_declarator_lang(lang) && cbm_c_reserved_func_name(text)) { + return NULL; + } if (text && strcmp(ts_node_type(name_node), "operator_cast") == 0) { char *paren = strchr(text, '('); if (paren) { diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 00e3d30dc..542615231 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -58,6 +58,37 @@ const char *cbm_enclosing_func_qn_cached(CBMExtractCtx *ctx, TSNode node); // enclosing-function attribution — drift between private copies caused #438. TSNode cbm_resolve_c_declarator_name_node(TSNode func_node); +// tree-sitter error recovery around an unknown leading macro: `API RetT name(...)` +// (`XXH_PUBLIC_API XXH64_hash_t XXH64(...)`, `JEMALLOC_ALWAYS_INLINE T *f(...)`) +// carries one identifier more than the declaration grammar admits, so the parser +// recovers in one of two shapes, and neither names the function: +// - C++ inserts a zero-width MISSING "::" and reads `RetT name` as the +// out-of-line name `RetT::name`; with a pointer return, the name side is a +// pointer_type_declarator that wraps the function_declarator; +// - C and C++ keep RetT as the declarator identifier and put the real name in +// an ERROR node directly before the parameter list. +// cbm_c_qualifier_is_recovered: true when `qid` (a qualified_identifier or +// scoped_identifier) is the first shape, i.e. its "::" token is MISSING. +// cbm_c_recovered_func_name: the real name node of the second shape for a +// function_declarator, or a null node when the declarator is not that shape. +// Shared by every C-family declarator walk (defs, call scope, params, C LSP) so +// the def QN and every caller QN agree on the recovered name. +bool cbm_c_qualifier_is_recovered(TSNode qid); +TSNode cbm_c_recovered_func_name(TSNode func_declarator); + +// True for a C keyword that error recovery can surface as a function "name" +// (`else if (a) (b) {` split by #ifdef reads as a definition named `if`). +// cbm_func_name_node_text returns NULL for these in the C-declarator grammars, +// so neither a def nor a call scope is minted; the C LSP applies the same test. +bool cbm_c_reserved_func_name(const char *name); + +// True for the languages whose `#define`s become Macro nodes: C, C++, CUDA, +// GLSL, Objective-C and ISPC. The one predicate for what follows from the C +// preprocessor: the `#macro` QN namespace (CBM_MACRO_QN_SUFFIX) and the +// `variants` property (CBMDefinition.variants), because #if branches are what +// lets one of these files define a name more than once. +bool cbm_is_c_preprocessor_lang(CBMLanguage lang); + // Convert a resolved function/method name node to its name string, normalizing a // C++ conversion-operator's `operator_cast` node (which spans the full // "operator bool() const") down to "operator bool". Shared by the defs and diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 74b97d69d..7ff93bd0d 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -4795,22 +4795,44 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { // Navigate declarator to find name and parameters TSNode cur = decl; + // Set after stepping through a recovered `RetT::name` (see + // cbm_c_qualifier_is_recovered): that path names the function with a + // type_identifier, and the "scope" is the return type, not a class. + bool recovered = false; for (int depth = 0; depth < 10 && !ts_node_is_null(cur); depth++) { const char *dk = ts_node_type(cur); if (strcmp(dk, "function_declarator") == 0) { TSNode fdecl = ts_node_child_by_field_name(cur, "declarator", 10); params_node = ts_node_child_by_field_name(cur, "parameters", 10); + // `API RetT name(...)`: the real name sits in an ERROR before the + // parameters; the declarator identifier is the return type. + TSNode real = cbm_c_recovered_func_name(cur); + if (!ts_node_is_null(real)) { + func_name = c_node_text(ctx, real); + break; + } cur = fdecl; continue; } - if (strcmp(dk, "pointer_declarator") == 0 || strcmp(dk, "reference_declarator") == 0) { + if (strcmp(dk, "pointer_declarator") == 0 || strcmp(dk, "reference_declarator") == 0 || + (recovered && strcmp(dk, "pointer_type_declarator") == 0)) { if (ts_node_named_child_count(cur) > 0) cur = ts_node_named_child(cur, ts_node_named_child_count(cur) - 1); else break; continue; } + if (recovered && strcmp(dk, "type_identifier") == 0) { + func_name = c_node_text(ctx, cur); + break; + } + if ((strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) && + cbm_c_qualifier_is_recovered(cur)) { + cur = ts_node_child_by_field_name(cur, "name", 4); + recovered = true; + continue; + } if (strcmp(dk, "qualified_identifier") == 0 || strcmp(dk, "scoped_identifier") == 0) { func_name = c_node_text(ctx, cur); // Check if this is a method (has Class:: prefix) @@ -4850,7 +4872,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) { break; } - if (!func_name || !func_name[0]) + // A keyword "name" is an error-recovery artifact (see cbm_c_reserved_func_name): + // the def extractor mints no node for it, so no caller QN may name it either. + if (!func_name || !func_name[0] || cbm_c_reserved_func_name(func_name)) return; // Build enclosing function QN diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index bd8b9b8a9..97b16be6e 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -275,6 +275,7 @@ static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) { cr_str(c, &d->impl_trait); cr_str(c, &d->http_client); cr_str(c, &d->http_base_url); + cr_str(c, &d->variants); } static void cr_walk_call(cr_ctx_t *c, CBMCall *call) { diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index e55eb28f0..5484e1a4f 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -837,6 +837,20 @@ void cbm_gbuf_set_next_id(cbm_gbuf_t *gb, int64_t next_id) { /* ── Node operations ─────────────────────────────────────────────── */ +/* First key of the canonical same-QN collision order: how strong a claim the + * entity has on a qualified name. A macro is the weakest: when a "Macro" and + * a definition share a QN, the definition, which carries the body, the + * signature and every edge, owns it whichever comes first. + * + * C-preprocessor macros no longer get here: their QN carries its own + * namespace (`...#macro`, CBM_MACRO_QN_SUFFIX) and cannot equal a definition's. + * The case left is Chialisp, whose `defmacro` / `defmac` defs are labelled + * "Macro" under a plain QN: a macro and a `defun` of one name in one file, or + * in a same-stem `.clsp` / `.clib` pair (one module QN). */ +static int gb_qn_claim_rank(const char *label) { + return (label && strcmp(label, "Macro") == 0) ? 0 : 1; +} + int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name, const char *qualified_name, const char *file_path, int start_line, int end_line, const char *properties_json) { @@ -880,8 +894,13 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name * of the same entity with new content) replaces the earlier one, * deterministically, because intra-file arrival order is fixed. A * full tie is the same entity re-upserted → refresh in place. - * Kind-disambiguated QNs (the real cure) remain a follow-up. */ - int c = strcmp(file_path ? file_path : "", existing->file_path ? existing->file_path : ""); + * Ahead of all of these, a definition beats a macro + * (gb_qn_claim_rank). Kind-disambiguated QNs (the real cure) remain + * a follow-up. */ + int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(label); + if (c == 0) { + c = strcmp(file_path ? file_path : "", existing->file_path ? existing->file_path : ""); + } if (c == 0) { c = existing->start_line - start_line; } @@ -1480,9 +1499,13 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing, * the node set (and every downstream consumer) run to run. Winner = * smallest file_path, then LARGEST start_line, then largest * name/label — one total order, commutative, scheduling-free; a full - * tie is the same entity → refresh from src. */ - int c = strcmp(sn->file_path ? sn->file_path : "", + * tie is the same entity → refresh from src. A definition beats a + * macro ahead of all of these (gb_qn_claim_rank). */ + int c = gb_qn_claim_rank(existing->label) - gb_qn_claim_rank(sn->label); + if (c == 0) { + c = strcmp(sn->file_path ? sn->file_path : "", existing->file_path ? existing->file_path : ""); + } if (c == 0) { c = existing->start_line - sn->start_line; } diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index e67cf4aac..c95cf5b12 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -8518,6 +8518,17 @@ static long node_resolution_score(const cbm_node_t *n) { label_rank = RES_RANK_OTHER; } } + /* Tie rule (CBM_MACRO_QN_SUFFIX): a name that is both a definition and a C + * macro -- a typedef or enumerator next to its rename macro -- resolves to + * the definition. The macro scores just under every other definition, and + * still above Module/File; without this a one-line typedef and its macro + * tie on span and the name reads as ambiguous. */ + size_t qn_len = n->qualified_name ? strlen(n->qualified_name) : 0; + size_t fence_len = sizeof(CBM_MACRO_QN_SUFFIX) - SKIP_ONE; + if (label_rank == RES_RANK_OTHER && qn_len > fence_len && + strcmp(n->qualified_name + qn_len - fence_len, CBM_MACRO_QN_SUFFIX) == 0) { + return RES_RANK_OTHER * (long)RES_LABEL_WEIGHT - SKIP_ONE; + } long span = (long)n->end_line - (long)n->start_line; if (span < 0) { span = 0; @@ -12871,6 +12882,19 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { int suffix_count = 0; cbm_store_find_nodes_by_qn_suffix(store, effective_project, qn, &suffix_nodes, &suffix_count); + /* Tier 3: the C-macro namespace. A macro's QN is `.#macro` + * (CBM_MACRO_QN_SUFFIX), out of reach of '%.X'. Tried only when no + * definition answered to the name above (tie rule), for a short name, a + * partial QN, or the macro's plain QN. */ + char macro_qn[CBM_SZ_512]; + int macro_len = snprintf(macro_qn, sizeof(macro_qn), "%s" CBM_MACRO_QN_SUFFIX, qn); + if (suffix_count == 0 && macro_len > 0 && (size_t)macro_len < sizeof(macro_qn)) { + cbm_store_free_nodes(suffix_nodes, suffix_count); + suffix_nodes = NULL; + cbm_store_find_nodes_by_qn_suffix(store, effective_project, macro_qn, &suffix_nodes, + &suffix_count); + } + if (suffix_count == SKIP_ONE) { copy_node(&suffix_nodes[0], &node); cbm_store_free_nodes(suffix_nodes, suffix_count); diff --git a/src/pipeline/lsp_resolve.h b/src/pipeline/lsp_resolve.h index 666f9fd55..a8ba015f3 100644 --- a/src/pipeline/lsp_resolve.h +++ b/src/pipeline/lsp_resolve.h @@ -920,24 +920,39 @@ static inline const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node_policy( if (direct) { return direct; } - if (project_name && project_name[0]) { - size_t proj_len = strlen(project_name); - if (!(strncmp(callee_qn, project_name, proj_len) == 0 && callee_qn[proj_len] == '.')) { - size_t callee_len = strlen(callee_qn); - if (proj_len <= SIZE_MAX - callee_len - 2U) { - size_t prefixed_len = proj_len + 1U + callee_len; - char *prefixed_qn = (char *)malloc(prefixed_len + 1U); - if (prefixed_qn) { - memcpy(prefixed_qn, project_name, proj_len); - prefixed_qn[proj_len] = '.'; - memcpy(prefixed_qn + proj_len + 1U, callee_qn, callee_len + 1U); - const cbm_gbuf_node_t *prefixed = cbm_gbuf_find_by_qn(gbuf, prefixed_qn); - free(prefixed_qn); - if (prefixed) { - return prefixed; - } + /* Exact retries, in tie-rule order (CBM_MACRO_QN_SUFFIX): the definition + * first -- the QN as given, then with the project prefix -- and only when + * neither names a node the C-preprocessor macro of that name, in the same + * two spellings. The macro case is a prototype the C resolver knows whose + * only node in the graph is its rename macro (`extern T f(...);` plus + * `#define f f_v2`). One buffer, ".#macro", serves all + * three retries. */ + size_t proj_len = (project_name && project_name[0]) ? strlen(project_name) : 0; + size_t callee_len = strlen(callee_qn); + size_t suffix_len = sizeof(CBM_MACRO_QN_SUFFIX) - 1U; + bool add_prefix = proj_len > 0 && !(strncmp(callee_qn, project_name, proj_len) == 0 && + callee_qn[proj_len] == '.'); + if (proj_len <= SIZE_MAX - callee_len - suffix_len - 2U) { + char *buf = (char *)malloc(proj_len + 1U + callee_len + suffix_len + 1U); + if (buf) { + char *own = buf + proj_len + 1U; /* callee_qn inside the buffer */ + if (proj_len > 0) { + memcpy(buf, project_name, proj_len); + } + buf[proj_len] = '.'; + memcpy(own, callee_qn, callee_len + 1U); + const cbm_gbuf_node_t *hit = add_prefix ? cbm_gbuf_find_by_qn(gbuf, buf) : NULL; + if (!hit) { + memcpy(own + callee_len, CBM_MACRO_QN_SUFFIX, suffix_len + 1U); + hit = cbm_gbuf_find_by_qn(gbuf, own); + if (!hit && add_prefix) { + hit = cbm_gbuf_find_by_qn(gbuf, buf); } } + free(buf); + if (hit) { + return hit; + } } } if (!allow_tail_match) { diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index f3a2cd948..c91b8a6c6 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -203,6 +203,24 @@ static void append_json_string(char *buf, size_t bufsize, size_t *pos, const cha *pos = p; } +/* Append an already-serialized JSON value verbatim: ,"key":. Atomic like + * append_json_string. For `variants`, which extraction builds as a JSON array + * of objects (cbm.h). Twin of pass_parallel.c -- keep both in sync. */ +static void append_json_raw(char *buf, size_t bufsize, size_t *pos, const char *key, + const char *json) { + if (!json || json[0] == '\0') { + return; + } + size_t required = strlen(key) + strlen(json) + PD_JSON_FIELD_OVERHEAD; + if (*pos + required + PD_ESC_SPACE > bufsize) { + return; /* whole field would not fit — skip it atomically */ + } + int w = snprintf(buf + *pos, bufsize - *pos, ",\"%s\":%s", key, json); + if (w > 0 && (size_t)w < bufsize - *pos) { + *pos += (size_t)w; + } +} + /* Append a JSON array of strings: ,"key":["a","b","c"]. Atomic like * append_json_string: emitted only if the whole array fits. */ static void append_json_str_array(char *buf, size_t bufsize, size_t *pos, const char *key, @@ -286,6 +304,9 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) } size_t pos = (size_t)n; append_json_string(buf, bufsize, &pos, "docstring", def->docstring); + /* Right after the docstring: the buffer is sized for exactly these two + * uncapped fields (pd_props_buf), so neither can be squeezed out. */ + append_json_raw(buf, bufsize, &pos, "variants", def->variants); append_json_string(buf, bufsize, &pos, "signature", def->signature); append_json_string(buf, bufsize, &pos, "return_type", def->return_type); append_json_string(buf, bufsize, &pos, "parent_class", def->parent_class); @@ -323,16 +344,21 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) } /* A def's properties buffer: CBM_SZ_2K for every other field plus the whole - * serialized docstring field, which has no length cap (a field that does not - * fit is dropped whole). Returns `stack` for a def without a docstring, or - * when the larger buffer cannot be allocated. Twin of pass_parallel.c -- keep - * both in sync. */ + * serialized docstring and variants fields, which have no length cap (a field + * that does not fit is dropped whole). Returns `stack` for a def with neither, + * or when the larger buffer cannot be allocated. Twin of pass_parallel.c -- + * keep both in sync. */ static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) { - if (!def->docstring || !def->docstring[0]) { + size_t need = *size; + if (def->docstring && def->docstring[0]) { + need += strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD; + } + if (def->variants && def->variants[0]) { + need += strlen("variants") + strlen(def->variants) + PD_JSON_FIELD_OVERHEAD; + } + if (need == *size) { return stack; } - size_t need = - *size + strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD; char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need); if (!buf) { return stack; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 0c12e6c4a..03c065d96 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -477,6 +477,24 @@ static void append_json_str_array(char *buf, size_t bufsize, size_t *pos, const *pos = p; } +/* Append an already-serialized JSON value verbatim: ,"key":. Atomic like + * append_json_string. For `variants`, which extraction builds as a JSON array + * of objects (cbm.h). Twin of pass_definitions.c -- keep both in sync. */ +static void append_json_raw(char *buf, size_t bufsize, size_t *pos, const char *key, + const char *json) { + if (!json || json[0] == '\0') { + return; + } + size_t required = strlen(key) + strlen(json) + PP_JSON_FIELD_OVERHEAD; + if (*pos + required + PP_ESC_SPACE > bufsize) { + return; /* whole field would not fit — skip it atomically */ + } + int w = snprintf(buf + *pos, bufsize - *pos, ",\"%s\":%s", key, json); + if (w > 0 && (size_t)w < bufsize - *pos) { + *pos += (size_t)w; + } +} + static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) { /* Complexity/loop/recursion metrics are meaningful only for Function/Method. * Gate the block so the millions of Macro/Field/Variable/Class/Enum nodes @@ -513,6 +531,9 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) } size_t pos = (size_t)n; append_json_string(buf, bufsize, &pos, "docstring", def->docstring); + /* Right after the docstring: the buffer is sized for exactly these two + * uncapped fields (pp_props_buf), so neither can be squeezed out. */ + append_json_raw(buf, bufsize, &pos, "variants", def->variants); append_json_string(buf, bufsize, &pos, "signature", def->signature); append_json_string(buf, bufsize, &pos, "return_type", def->return_type); append_json_string(buf, bufsize, &pos, "parent_class", def->parent_class); @@ -698,16 +719,21 @@ typedef struct { enum { PP_OVERSIZED_WARN_MAX = 32 }; /* A def's properties buffer: CBM_SZ_2K for every other field plus the whole - * serialized docstring field, which has no length cap (a field that does not - * fit is dropped whole). Returns `stack` for a def without a docstring, or - * when the larger buffer cannot be allocated. Twin of pass_definitions.c -- + * serialized docstring and variants fields, which have no length cap (a field + * that does not fit is dropped whole). Returns `stack` for a def with neither, + * or when the larger buffer cannot be allocated. Twin of pass_definitions.c -- * keep both in sync. */ static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) { - if (!def->docstring || !def->docstring[0]) { + size_t need = *size; + if (def->docstring && def->docstring[0]) { + need += strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD; + } + if (def->variants && def->variants[0]) { + need += strlen("variants") + strlen(def->variants) + PP_JSON_FIELD_OVERHEAD; + } + if (need == *size) { return stack; } - size_t need = - *size + strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD; char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need); if (!buf) { return stack; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 28daa5434..ae741fbbe 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -806,8 +806,14 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr cbm_file_hash_t **out, int *out_count); /* Compatibility contract persisted in coverage metadata. Increment when a - * graph/manifest semantic change makes prior exact-input indexes unsafe. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 3 }; + * graph/manifest semantic change makes prior exact-input indexes unsafe. + * 4: C-family node identities changed. A preprocessor macro's QN ends in + * "#macro"; unscoped C/C++/Objective-C enumerators are `.` + * (the enum name is no longer a segment); typedef names, anonymous-enum + * constants and macro-prefixed functions are nodes; a bodyless + * `struct X` is no node. An index written before this holds the old + * QNs for every unchanged file, so it is rebuilt in full once. */ +enum { CBM_SEMANTIC_INDEX_VERSION = 4 }; typedef struct { cbm_gbuf_t *gbuf; diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index bdf6f5944..5b5e79df0 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -947,7 +947,9 @@ void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified const char *derived = simple_name(qualified_name); index_under_name(r, derived, owned_qn); /* '#' is a QN fence, and extract_defs.c's rust_cfg_qualified_name is the - * only thing in the tree that mints one today. A grammar that starts + * only thing in the tree that mints one on a REGISTERED symbol today (a C + * macro's QN ends in "#macro", but Macro is no registry label, so it never + * gets here). A grammar that starts * minting a '#' opts into this second key by doing so, whatever it means by * the fence: its symbols become reachable under the passed name as well, * and they share that name's bucket with everything else filed under it. */ diff --git a/src/store/store.c b/src/store/store.c index d66ef0ffb..668013544 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -421,7 +421,18 @@ static int init_schema(cbm_store_t *s) { "CASE WHEN properties LIKE '%\"docstring\"%' AND json_valid(properties) " \ "THEN json_extract(properties, '$.docstring') END" -enum { FTS_SQL_BUF = 512 }; +/* nodes_fts.qualified_name: the QN without the C-macro fence. A macro's QN is + * `.NAME#macro` (CBM_MACRO_QN_SUFFIX in internal/cbm/cbm.h), and `#` + * separates tokens, so the fence would add the word "macro" to every macro + * node a second time (the label column already says Macro). On a C repository + * that is thousands of rows outscoring the few nodes that carry the word in + * their NAME, enough to push those out of the BM25 candidate window. */ +#define FTS_QN_EXPR \ + "CASE WHEN label = 'Macro' AND substr(qualified_name, -6) = '#macro' " \ + "THEN substr(qualified_name, 1, length(qualified_name) - 6) " \ + "ELSE qualified_name END" + +enum { FTS_SQL_BUF = 768 }; /* Does nodes_fts carry the `body` column? A database created by an older * build has only the four identifier columns, and CREATE VIRTUAL TABLE IF NOT @@ -446,7 +457,7 @@ static int fts_backfill_try(cbm_store_t *s, const char *project, int64_t after_i char sql[FTS_SQL_BUF]; int n = snprintf(sql, sizeof(sql), "INSERT INTO nodes_fts (rowid, name, qualified_name, label, file_path%s)" - " SELECT id, %s, qualified_name, label, file_path%s FROM nodes%s;", + " SELECT id, %s, " FTS_QN_EXPR ", label, file_path%s FROM nodes%s;", with_body ? ", body" : "", camel ? "cbm_camel_split(name)" : "name", with_body ? ", " FTS_BODY_EXPR : "", project ? " WHERE project = ?1 AND id > ?2" : ""); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index fe46d7246..55e203144 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -183,12 +183,13 @@ TEST(extract_ts_factory_object_methods_issue341) { * * The bound is derived, not tuned. Measured on this source, total_alloc was * 365984 before the scratch arena and is 87456 after, exactly that difference. - * #1916 added two pointers to CBMDefinition (240 -> 256 bytes), so it now - * measures 87968: 8192 is the defs item array at GROW_ARRAY's starting - * capacity of 32 times sizeof(CBMDefinition) 256, and the other 79776 is - * everything else this file's extraction interns; none of it is traversal - * scratch. So the bound sits above 87968 with room and a factor of four below - * the 365984 the scratch stacks cost. + * #1916 added two pointers to CBMDefinition (240 -> 256 bytes) and the C + * `variants` list one more (264), so it now measures 88224: 8448 is the defs + * item array at GROW_ARRAY's starting capacity of 32 times + * sizeof(CBMDefinition) 264, and the other 79776 is everything else this + * file's extraction interns; none of it is traversal scratch. So the bound + * sits above 88224 with room and a factor of four below the 365984 the + * scratch stacks cost. * * It is a byte budget, not a proof of lifetime; that is * extract_traversal_stacks_come_from_ctx_scratch_issue2010 in test_mem.c. */ @@ -6658,6 +6659,740 @@ TEST(extract_c_export_macro_recovery_issue1989) { PASS(); } +/* ── PR C1: C graph defects behind the C/Doxygen doc-link audit ──────── */ + +static const CBMDefinition *c1_def(CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +/* Caller QN of the first raw-pass call to `callee`, or "" when there is none. */ +static const char *c1_call_scope(CBMFileResult *r, const char *callee) { + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (c->callee_name && strcmp(c->callee_name, callee) == 0 && + c->source_origin == CBM_SOURCE_ORIGIN_RAW) { + return c->enclosing_func_qn ? c->enclosing_func_qn : ""; + } + } + return ""; +} + +/* Caller QN of the first C-LSP resolution whose callee QN ends in ".". */ +static const char *c1_lsp_caller(CBMFileResult *r, const char *leaf) { + size_t ll = strlen(leaf); + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + size_t cl = rc->callee_qn ? strlen(rc->callee_qn) : 0; + if (cl > ll && rc->callee_qn[cl - ll - 1] == '.' && + strcmp(rc->callee_qn + cl - ll, leaf) == 0) { + return rc->caller_qn ? rc->caller_qn : ""; + } + } + return ""; +} + +/* xxhash.h shape: `XXH_PUBLIC_API name(...)` has one leading identifier more than + * the grammar admits. tree-sitter-cpp (every .h is C++) recovers it as `RetT::name` with a + * zero-width MISSING "::" (a Method of the return type), with a pointer return as + * `RetT::*name` (no name at all: the function was dropped), or keeps RetT as the name and + * puts the real one in an ERROR before the parameters. The implementation sits behind a + * guard the preprocessed pass does not take, as in xxhash.h, so only the raw parse sees + * it. Def QN, call-scope QN and C-LSP caller QN must all name the real function. */ +static const char *C1_MACRO_PREFIX_SRC = + "#if defined(XXH_IMPLEMENTATION)\n" + "XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)\n" + "{\n" + " XXH_free(statePtr);\n" + " return XXH_OK;\n" + "}\n" + "\n" + "XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)\n" + "{\n" + " return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));\n" + "}\n" + "\n" + "XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)\n" + "{\n" + " return XXH32_endian_align(input, len, seed);\n" + "}\n" + "#endif\n"; + +TEST(extract_cpp_macro_prefixed_function_names_c1) { + CBMFileResult *r = extract(C1_MACRO_PREFIX_SRC, CBM_LANG_CPP, "p", "xxhash.h"); + ASSERT_NOT_NULL(r); + char want[256]; + + const CBMDefinition *free_state = c1_def(r, "Function", "XXH32_freeState"); + ASSERT_NOT_NULL(free_state); + ASSERT_EQ(count_defs_named(r, "Method", "XXH32_freeState"), 0); + snprintf(want, sizeof(want), "%s.XXH32_freeState", r->module_qn); + ASSERT_STR_EQ(free_state->qualified_name, want); + ASSERT_STR_EQ(c1_call_scope(r, "XXH_free"), want); + + const CBMDefinition *create = c1_def(r, "Function", "XXH32_createState"); + ASSERT_NOT_NULL(create); + ASSERT_EQ((int)create->start_line, 8); + ASSERT_EQ((int)create->end_line, 11); + ASSERT_NOT_NULL(create->signature); + + const CBMDefinition *xxh32 = c1_def(r, "Function", "XXH32"); + ASSERT_NOT_NULL(xxh32); + ASSERT_FALSE(has_def_any(r, "XXH32_hash_t")); + ASSERT_STR_EQ(c1_call_scope(r, "XXH32_endian_align"), xxh32->qualified_name); + cbm_free_result(r); + PASS(); +} + +/* The C-LSP walks declarators on its own (c_process_function); its caller QN must + * name the recovered function too, or the exact-equality LSP join drops the + * type-aware resolution of every call inside it (the call falls back to a textual + * strategy). jemalloc-style prefix: not an export-macro candidate, so the + * preprocessed pass keeps the same misparse and cannot mask the raw one. */ +TEST(extract_cpp_macro_prefixed_lsp_caller_c1) { + CBMFileResult *r = extract("static int je_helper(int v) { return v; }\n" + "JE_ALWAYS_INLINE je_errorcode je_reset(je_state_t *st)\n" + "{\n" + " return je_helper(1);\n" + "}\n" + "JE_ALWAYS_INLINE je_hash_t je_hash (const void *in, size_t n)\n" + "{\n" + " return je_helper(2);\n" + "}\n", + CBM_LANG_CPP, "p", "je.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *reset = c1_def(r, "Function", "je_reset"); + const CBMDefinition *hash = c1_def(r, "Function", "je_hash"); + ASSERT_NOT_NULL(reset); + ASSERT_NOT_NULL(hash); + int from_reset = 0; + int from_hash = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const char *caller = r->resolved_calls.items[i].caller_qn; + from_reset += caller && strcmp(caller, reset->qualified_name) == 0; + from_hash += caller && strcmp(caller, hash->qualified_name) == 0; + } + ASSERT_GT(from_reset, 0); + ASSERT_GT(from_hash, 0); + ASSERT_STR_EQ(c1_lsp_caller(r, "je_helper"), reset->qualified_name); + cbm_free_result(r); + PASS(); +} + +/* The same recovery in the C grammar (lua's LUA_API): named after the return type + * before. A function-like macro invocation that only looks like a declarator + * (`... TEST_BEGIN (test_alignment) {`) has an untyped "parameter" no definition can + * have, so its macro name is never adopted as the function name. */ +TEST(extract_c_macro_prefixed_function_name_c1) { + CBMFileResult *r = + extract("#if defined(LUA_CORE)\n" + "LUA_API lua_CFunction lua_atpanic (lua_State *L, lua_CFunction panicf) {\n" + " lua_CFunction old = G(L)->panic;\n" + " return old;\n" + "}\n" + "\n" + "XXH_TEST_API XXH32_hash_t TEST_BEGIN (test_alignment) {\n" + " helper(1);\n" + "}\n" + "#endif\n", + CBM_LANG_C, "p", "lapi.c"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(c1_def(r, "Function", "lua_atpanic")); + ASSERT_FALSE(has_def_any(r, "lua_CFunction")); + ASSERT_FALSE(has_def_any(r, "TEST_BEGIN")); + cbm_free_result(r); + PASS(); +} + +/* `struct X` / `enum X` / `class X` without a body is a reference, not a definition: + * no Class/Enum def, so nothing competes with the real definition for its QN (a later + * or smaller-path reference used to take the node: curl 132 structs, kernel + * task_struct, redis redisServer) and no phantom type node appears in every file that + * merely mentions the type (`struct timeval`). */ +TEST(extract_c_tag_reference_is_not_a_definition_c1) { + CBMFileResult *r = extract("struct node {\n" + " int value;\n" + " struct node *next;\n" + "};\n" + "struct node *head;\n" + "struct timeval tv;\n" + "enum color { RED, GREEN };\n" + "enum color paint(enum color c);\n" + "void walk(struct node *n);\n" + "struct opaque;\n", + CBM_LANG_C, "p", "list.c"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Class", "node"), 1); + const CBMDefinition *node = c1_def(r, "Class", "node"); + ASSERT_EQ((int)node->start_line, 1); + ASSERT_EQ((int)node->end_line, 4); + ASSERT_EQ(count_defs_named(r, "Enum", "color"), 1); + ASSERT_EQ((int)c1_def(r, "Enum", "color")->start_line, 7); + ASSERT_FALSE(has_def_any(r, "timeval")); + ASSERT_FALSE(has_def_any(r, "opaque")); + cbm_free_result(r); + + /* .h is parsed as C++: a forward declaration before the definition. */ + r = extract("class Widget;\n" + "struct Config;\n" + "Widget *make_widget(struct Config *cfg);\n" + "class Widget {\n" + " int size;\n" + "};\n", + CBM_LANG_CPP, "p", "widget.h"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Class", "Widget"), 1); + ASSERT_EQ((int)c1_def(r, "Class", "Widget")->start_line, 4); + ASSERT_FALSE(has_def_any(r, "Config")); + cbm_free_result(r); + PASS(); +} + +/* tree-sitter can give up on a whole region and leave the definition head as loose + * ERROR tokens (kernel include/linux/sched.h: `struct task_struct {` at 826 is never a + * struct_specifier). Its only nodes were then the references (642 won); with + * references no longer minting defs the head itself must be recovered, spanning to + * its matching brace counted over the first branch of each #if group. */ +TEST(extract_c_struct_head_in_error_region_recovered_c1) { + CBMFileResult *r = extract("#ifndef SCHED_H\n" + "#define SCHED_H\n" + "struct task_struct;\n" + "typedef struct task_struct *(*pick_f)(int cpu);\n" + "\n" + "/* Per-task scheduler state. */\n" + "struct task_struct {\n" + "\tint state;\n" + "#ifdef CONFIG_SMP\n" + "\tstruct {\n" + "#else\n" + "\tunion {\n" + "#endif\n" + "\t\tint on_cpu;\n" + "\t};\n" + "\tint prio;\n" + "};\n" + "#endif\n", + CBM_LANG_CPP, "p", "sched.h"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Class", "task_struct"), 1); + const CBMDefinition *ts = c1_def(r, "Class", "task_struct"); + ASSERT_EQ((int)ts->start_line, 7); + ASSERT_EQ((int)ts->end_line, 17); + ASSERT_NOT_NULL(c1_def(r, "Type", "pick_f")); + cbm_free_result(r); + PASS(); +} + +/* A stray `;` after an if-block leaves the following `else if (...) {...}` without its + * `if`; inside a function the raw parse already lost (whole-file guard + split brace), + * error recovery reads it as a DEFINITION: type `else`, name `if`. A C keyword is never + * a function name (curl lib/vtls/openssl.c:4242, redis deps/tre/lib/tre-parse.c:1315). */ +TEST(extract_c_keyword_is_never_a_function_name_c1) { + CBMFileResult *r = extract("#ifdef USE_OPENSSL\n" + "static int check(int lib, int reason)\n" + "{\n" + " int result = 0;\n" + " if(lib == 1) {\n" + "#ifndef HAVE_BORINGSSL_LIKE\n" + " result = 1; {\n" + "#else\n" + " result = 2; {\n" + "#endif\n" + " }\n" + " };\n" + " else if(reason == 2) {\n" + " result = trace_retry(reason);\n" + " }\n" + " return result;\n" + "}\n" + "#endif\n", + CBM_LANG_C, "p", "openssl.c"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(has_def_any(r, "if")); + ASSERT_TRUE(has_call(r, "trace_retry")); + cbm_free_result(r); + PASS(); +} + +/* Preprocessor-split code the raw parse cannot read: `#ifndef X (a)) { #else (b)) { + * #endif` leaves an extra `{`, the file-level ERROR swallows the function header and + * the function gets no node. The #961 rescue re-parses the preprocessed text, but a + * whole-file `#ifdef USE_OPENSSL` with no build defines leaves it nothing to parse. + * The first-branch projection restores it (curl ossl_connect_step2, + * Curl_ossl_ctx_init, mbed_configure_ssl, ...: all 11 curl misses). */ +TEST(extract_c_function_lost_to_split_branches_restored_c1) { + CBMFileResult *r = extract("#ifdef USE_OPENSSL\n" + "static int ossl_step(int lib, int reason)\n" + "{\n" + " int result = 0;\n" + " if(lib == 1) {\n" + " result = 1;\n" + " }\n" + " else if((lib == 2) &&\n" + "#ifndef HAVE_BORINGSSL_LIKE\n" + " (reason == 3)) {\n" + "#else\n" + " (reason == 4)) {\n" + "#endif\n" + " result = trace_retry(reason);\n" + " }\n" + " return result;\n" + "}\n" + "\n" + "int ossl_after(void)\n" + "{\n" + " return ossl_step(1, 2);\n" + "}\n" + "#endif\n", + CBM_LANG_C, "p", "openssl.c"); + ASSERT_NOT_NULL(r); + const CBMDefinition *step = c1_def(r, "Function", "ossl_step"); + ASSERT_NOT_NULL(step); + ASSERT_EQ((int)step->start_line, 2); + ASSERT_EQ((int)step->end_line, 17); + ASSERT_EQ(count_defs_named(r, "Function", "ossl_after"), 1); + cbm_free_result(r); + PASS(); +} + +/* C typedefs carry no `name` field (the alias lives in the declarators), so no typedef + * ever became a node and an anonymous `typedef struct {...} Name;` lost its fields and + * enum constants too. xxhash: `@ref XXH32_state_t` could only resolve to the namespace + * rename macro `#define XXH32_state_t XXH_IPREF(XXH32_state_t)`. */ +TEST(extract_c_typedef_names_are_definitions_c1) { + CBMFileResult *r = extract("typedef struct XXH32_state_s XXH32_state_t;\n" + "typedef struct {\n" + " unsigned char digest[4];\n" + "} XXH32_canonical_t;\n" + "typedef enum { XXH_OK = 0, XXH_ERROR = 1 } XXH_errorcode;\n" + "typedef unsigned int XXH32_hash_t;\n" + "typedef int (*cmp_fn)(const void *a, const void *b);\n" + "typedef struct opaque_s opaque_s;\n" + "typedef struct node node;\n" + "struct node { int v; };\n" + "typedef struct named_s { int y; } named_t;\n" + "typedef struct same { int z; } same;\n", + CBM_LANG_C, "p", "xxhash.c"); + ASSERT_NOT_NULL(r); + char want[256]; + ASSERT_NOT_NULL(c1_def(r, "Type", "XXH32_state_t")); + const CBMDefinition *canon = c1_def(r, "Class", "XXH32_canonical_t"); + ASSERT_NOT_NULL(canon); + ASSERT_EQ((int)canon->start_line, 2); + snprintf(want, sizeof(want), "%s.XXH32_canonical_t.digest", r->module_qn); + ASSERT_TRUE(has_def_qn(r, want)); + ASSERT_NOT_NULL(c1_def(r, "Enum", "XXH_errorcode")); + snprintf(want, sizeof(want), "%s.XXH_OK", r->module_qn); + ASSERT_TRUE(has_def_qn(r, want)); + ASSERT_NOT_NULL(c1_def(r, "Type", "XXH32_hash_t")); + ASSERT_NOT_NULL(c1_def(r, "Type", "cmp_fn")); + /* identity alias of an incomplete struct: often the only declaration of an + * opaque handle type */ + ASSERT_NOT_NULL(c1_def(r, "Type", "opaque_s")); + /* identity alias of a struct defined in the same file: the definition owns it */ + ASSERT_EQ(count_defs_named(r, "Class", "node"), 1); + ASSERT_EQ(count_defs_named(r, "Type", "node"), 0); + ASSERT_NOT_NULL(c1_def(r, "Class", "named_s")); + ASSERT_NOT_NULL(c1_def(r, "Type", "named_t")); + ASSERT_EQ(count_defs_named(r, "Class", "same"), 1); + ASSERT_EQ(count_defs_named(r, "Type", "same"), 0); + cbm_free_result(r); + PASS(); +} + +static const CBMDefinition *c1_def_qn(CBMFileResult *r, const char *qn) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->qualified_name && strcmp(d->qualified_name, qn) == 0) { + return d; + } + } + return NULL; +} + +/* The def with this label and name that starts on `line`. */ +static const CBMDefinition *c1_def_at(CBMFileResult *r, const char *label, const char *name, + int line) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0 && + (int)d->start_line == line) { + return d; + } + } + return NULL; +} + +/* A preprocessor macro's QN is `.#macro` (CBM_MACRO_QN_SUFFIX); name and + * label stay. C keeps macros apart from ordinary identifiers, so a macro and the + * function, type or enumerator of the same name are two entities and need two QNs: the + * namespace rename `#define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)` held the QN of the + * function XXH32 (110 of 174 audited xxhash doc links ended on such a macro), and the + * #else stand-in `#define match_proxy(a) 1` replaced the function it stands in for + * (curl, 41 functions). Every language routed through the #define path is covered, + * and in each of them a macro defined twice carries `variants`: the namespace and the + * list follow one language predicate. */ +TEST(extract_c_macro_qn_is_fenced_c1) { + static const struct { + CBMLanguage lang; + const char *file; + } cases[] = {{CBM_LANG_C, "cfg.c"}, {CBM_LANG_CPP, "cfg.h"}, + {CBM_LANG_OBJC, "cfg.m"}, {CBM_LANG_CUDA, "cfg.cu"}, + {CBM_LANG_GLSL, "cfg.glsl"}, {CBM_LANG_ISPC, "cfg.ispc"}}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = extract("#define MAX_LEN 10\n" + "#define SQUARE(x) ((x) * (x))\n" + "#define MAX_LEN 20\n", + cases[i].lang, "p", cases[i].file); + ASSERT_NOT_NULL(r); + char want[256]; + const CBMDefinition *max_len = c1_def(r, "Macro", "MAX_LEN"); + ASSERT_NOT_NULL(max_len); + snprintf(want, sizeof(want), "%s.MAX_LEN" CBM_MACRO_QN_SUFFIX, r->module_qn); + ASSERT_STR_EQ(max_len->qualified_name, want); + const CBMDefinition *square = c1_def(r, "Macro", "SQUARE"); + ASSERT_NOT_NULL(square); + snprintf(want, sizeof(want), "%s.SQUARE#macro", r->module_qn); + ASSERT_STR_EQ(square->qualified_name, want); + /* the plain QN is free for a definition of that name */ + snprintf(want, sizeof(want), "%s.MAX_LEN", r->module_qn); + ASSERT_NULL(c1_def_qn(r, want)); + const CBMDefinition *redefined = c1_def_at(r, "Macro", "MAX_LEN", 3); + ASSERT_NOT_NULL(redefined); + ASSERT_NOT_NULL(redefined->variants); + cbm_free_result(r); + } + + CBMFileResult *r = extract("#ifdef USE_PROXY\n" + "int match_proxy(int a) { return a; }\n" + "#else\n" + "#define match_proxy(a) 1\n" + "#endif\n", + CBM_LANG_C, "p", "proxy.c"); + ASSERT_NOT_NULL(r); + const CBMDefinition *fn = c1_def(r, "Function", "match_proxy"); + const CBMDefinition *mac = c1_def(r, "Macro", "match_proxy"); + ASSERT_NOT_NULL(fn); + ASSERT_NOT_NULL(mac); + char want[256]; + snprintf(want, sizeof(want), "%s.match_proxy", r->module_qn); + ASSERT_STR_EQ(fn->qualified_name, want); + snprintf(want, sizeof(want), "%s.match_proxy#macro", r->module_qn); + ASSERT_STR_EQ(mac->qualified_name, want); + cbm_free_result(r); + PASS(); +} + +/* The enumerators of a C enum live in the scope that holds the enum, so their QN is + * `.`: the enum's own name is not a segment. That is how a reference + * spells them (`XXH_OK`, never `XXH_errorcode.XXH_OK`), and it gives the constants of + * an anonymous enum, which has no name to nest under, a QN at all. `parent_class` + * (the enum's QN) keeps the membership; an anonymous enum has no node to point at. */ +TEST(extract_c_enumerators_are_flat_c1) { + CBMFileResult *r = extract("enum color { RED, GREEN = 2 };\n" + "enum { ANON_A, ANON_B };\n" + "typedef enum { TD_A, TD_B } td_t;\n", + CBM_LANG_C, "p", "e.c"); + ASSERT_NOT_NULL(r); + char qn[256]; + char parent[256]; + + snprintf(qn, sizeof(qn), "%s.RED", r->module_qn); + snprintf(parent, sizeof(parent), "%s.color", r->module_qn); + const CBMDefinition *red = c1_def_qn(r, qn); + ASSERT_NOT_NULL(red); + ASSERT_STR_EQ(red->label, "Variable"); + ASSERT_NOT_NULL(red->parent_class); + ASSERT_STR_EQ(red->parent_class, parent); + snprintf(qn, sizeof(qn), "%s.GREEN", r->module_qn); + ASSERT_NOT_NULL(c1_def_qn(r, qn)); + snprintf(qn, sizeof(qn), "%s.color.RED", r->module_qn); + ASSERT_NULL(c1_def_qn(r, qn)); + ASSERT_NOT_NULL(c1_def_qn(r, parent)); /* the Enum itself keeps its QN */ + + snprintf(qn, sizeof(qn), "%s.ANON_A", r->module_qn); + const CBMDefinition *anon = c1_def_qn(r, qn); + ASSERT_NOT_NULL(anon); + ASSERT_STR_EQ(anon->label, "Variable"); + ASSERT_NULL(anon->parent_class); + snprintf(qn, sizeof(qn), "%s.ANON_B", r->module_qn); + ASSERT_NOT_NULL(c1_def_qn(r, qn)); + + snprintf(qn, sizeof(qn), "%s.TD_A", r->module_qn); + snprintf(parent, sizeof(parent), "%s.td_t", r->module_qn); + const CBMDefinition *td = c1_def_qn(r, qn); + ASSERT_NOT_NULL(td); + ASSERT_NOT_NULL(td->parent_class); + ASSERT_STR_EQ(td->parent_class, parent); + ASSERT_EQ(count_defs_named(r, "Variable", "TD_A"), 1); + cbm_free_result(r); + PASS(); +} + +/* C++: an unscoped enum's enumerators belong to the enclosing scope (module, namespace + * or class), whatever nesting holds the enum; `enum class` and `enum struct` are + * scopes of their own and keep `.`. */ +TEST(extract_cpp_enum_scoping_c1) { + CBMFileResult *r = extract("enum Color { RED, GREEN };\n" + "namespace gfx {\n" + "enum Blend { ADD, MULTIPLY };\n" + "class Brush {\n" + "public:\n" + " enum Shape { ROUND, SQUARE };\n" + "};\n" + "}\n" + "enum class Mode { FAST, SLOW };\n" + "enum struct Level { LOW, HIGH };\n", + CBM_LANG_CPP, "p", "shapes.hpp"); + ASSERT_NOT_NULL(r); + static const struct { + const char *qn_tail; + const char *parent_tail; + } want[] = {{"RED", "Color"}, + {"GREEN", "Color"}, + {"gfx.ADD", "gfx.Blend"}, + {"gfx.MULTIPLY", "gfx.Blend"}, + {"gfx.Brush.ROUND", "gfx.Brush.Shape"}, + {"gfx.Brush.SQUARE", "gfx.Brush.Shape"}, + {"Mode.FAST", "Mode"}, + {"Mode.SLOW", "Mode"}, + {"Level.LOW", "Level"}, + {"Level.HIGH", "Level"}}; + for (size_t i = 0; i < sizeof(want) / sizeof(want[0]); i++) { + char qn[256]; + char parent[256]; + snprintf(qn, sizeof(qn), "%s.%s", r->module_qn, want[i].qn_tail); + snprintf(parent, sizeof(parent), "%s.%s", r->module_qn, want[i].parent_tail); + const CBMDefinition *d = c1_def_qn(r, qn); + if (!d) { + fprintf(stderr, " [c1] no def with QN %s\n", qn); + } + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->label, "Variable"); + ASSERT_NOT_NULL(d->parent_class); + ASSERT_STR_EQ(d->parent_class, parent); + } + char qn[256]; + snprintf(qn, sizeof(qn), "%s.Color.RED", r->module_qn); + ASSERT_NULL(c1_def_qn(r, qn)); + snprintf(qn, sizeof(qn), "%s.FAST", r->module_qn); + ASSERT_NULL(c1_def_qn(r, qn)); + cbm_free_result(r); + PASS(); +} + +/* A named and an anonymous enum inside a struct, and an anonymous one inside a union: + * one C-looking source, read under each C-family language by the two tests below. */ +static const char *const C1_ENUM_IN_STRUCT_SRC = "struct conn {\n" + " enum state { ST_IDLE, ST_BUSY } st;\n" + " enum { KIND_A, KIND_B } kind;\n" + " int fd;\n" + "};\n" + "union box {\n" + " enum { BOX_A, BOX_B } tag;\n" + " int v;\n" + "};\n"; + +/* Is `[.].` the ONE Variable def of that name, with parent_class + * `.` (NULL: no parent_class)? */ +static bool c1_enumerator_at(CBMFileResult *r, const char *scope, const char *name, + const char *parent_tail) { + char qn[256]; + snprintf(qn, sizeof(qn), "%s%s%s.%s", r->module_qn, scope[0] ? "." : "", scope, name); + const CBMDefinition *d = c1_def_qn(r, qn); + if (!d || !d->label || strcmp(d->label, "Variable") != 0 || + count_defs_named(r, "Variable", name) != 1) { + const CBMDefinition *other = c1_def(r, "Variable", name); + fprintf(stderr, " [c1] want the one Variable %s, the Variable of that name is %s\n", qn, + other ? other->qualified_name : "(none)"); + return false; + } + if (!parent_tail) { + return d->parent_class == NULL; + } + char parent[256]; + snprintf(parent, sizeof(parent), "%s.%s", r->module_qn, parent_tail); + return d->parent_class && strcmp(d->parent_class, parent) == 0; +} + +/* C has no struct scope for ordinary identifiers: an enum declared inside a struct or + * union puts its enumerators in the scope around it, and the code names them + * unqualified (curl lib/cf-h1-proxy.c: `ts->keepon = KEEPON_CONNECT;`). In C and + * Objective-C the struct is therefore no segment of a flat enumerator's QN, named + * enum or anonymous; `parent_class` still names a named enum. */ +TEST(extract_c_enum_inside_struct_is_file_scope_c1) { + static const struct { + CBMLanguage lang; + const char *file; + } cases[] = {{CBM_LANG_C, "link.c"}, {CBM_LANG_OBJC, "link.m"}}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = extract(C1_ENUM_IN_STRUCT_SRC, cases[i].lang, "p", cases[i].file); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(c1_enumerator_at(r, "", "ST_IDLE", "conn.state")); + ASSERT_TRUE(c1_enumerator_at(r, "", "ST_BUSY", "conn.state")); + ASSERT_TRUE(c1_enumerator_at(r, "", "KIND_A", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "", "KIND_B", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "", "BOX_A", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "", "BOX_B", NULL)); + cbm_free_result(r); + } + PASS(); +} + +/* C++ does scope them: `conn::ST_IDLE`. The enclosing class stays the scope in C++ and + * CUDA -- and so in every .h, which is parsed as C++ -- for the same source. This pins + * the other side of the rule above: the C treatment must not reach C++. */ +TEST(extract_cpp_enum_inside_class_keeps_class_scope_c1) { + static const struct { + CBMLanguage lang; + const char *file; + } cases[] = {{CBM_LANG_CPP, "link.h"}, {CBM_LANG_CUDA, "link.cu"}}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = extract(C1_ENUM_IN_STRUCT_SRC, cases[i].lang, "p", cases[i].file); + ASSERT_NOT_NULL(r); + ASSERT_TRUE(c1_enumerator_at(r, "conn", "ST_IDLE", "conn.state")); + ASSERT_TRUE(c1_enumerator_at(r, "conn", "ST_BUSY", "conn.state")); + ASSERT_TRUE(c1_enumerator_at(r, "conn", "KIND_A", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "conn", "KIND_B", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "box", "BOX_A", NULL)); + ASSERT_TRUE(c1_enumerator_at(r, "box", "BOX_B", NULL)); + cbm_free_result(r); + } + PASS(); +} + +/* A macro inside an enumerator list is not enumerator grammar; the parser keeps every + * bare identifier it can as an `enumerator` and wraps the rest in ERROR nodes. curl.h + * declares its options as `CURLOPT(CURLOPT_URL, CURLOPTTYPE_STRINGPOINT, 2),`: the + * second argument came back as a constant 115 times and, flattened, took the plain QN + * of the macro `CURLOPTTYPE_LONG`. `X CURL_DEPRECATED(7.55.0, "Use Y") = BASE + 16,` + * turned BASE and the words of the message into constants. Only the enumerator that + * opens a slot is a constant; the comment above the macro call documents it. */ +TEST(extract_c_enum_macro_wrapped_list_c1) { + CBMFileResult *r = extract("typedef enum {\n" + " /* doc of the first option */\n" + " OPT(OPT_FIRST, TYPE_LONG, 1),\n" + " OPT(OPT_SECOND, TYPE_STRING, 2),\n" + " OPT_THIRD DEPRECATED(1.0, \"Use OPT_FIRST instead\")\n" + " = BASE_LONG + 3,\n" + " OPT_LAST\n" + "} option_t;\n", + CBM_LANG_C, "p", "opts.c"); + ASSERT_NOT_NULL(r); + static const char *const real[] = {"OPT_FIRST", "OPT_SECOND", "OPT_THIRD", "OPT_LAST"}; + for (size_t i = 0; i < sizeof(real) / sizeof(real[0]); i++) { + ASSERT_EQ(count_defs_named(r, "Variable", real[i]), 1); + } + static const char *const bogus[] = {"TYPE_LONG", "TYPE_STRING", "BASE_LONG", + "Use", "instead", "OPT"}; + for (size_t i = 0; i < sizeof(bogus) / sizeof(bogus[0]); i++) { + ASSERT_EQ(count_defs_named(r, "Variable", bogus[i]), 0); + } + const CBMDefinition *first = c1_def(r, "Variable", "OPT_FIRST"); + ASSERT_NOT_NULL(first->docstring); + ASSERT_NOT_NULL(strstr(first->docstring, "doc of the first option")); + ASSERT_NULL(c1_def(r, "Variable", "OPT_SECOND")->docstring); + cbm_free_result(r); + PASS(); +} + +/* One file may define a QN several times under one label: both branches of an #if, a + * macro redefined per platform. The graph keeps one node per QN (the last by start + * line), and before this the other definitions left no trace (curl 123 groups, redis + * 87). The definition the graph keeps carries `variants`: every span of the group, + * the kept one included, sorted by start line. A name defined once carries nothing. */ +TEST(extract_c_variants_lists_same_file_duplicates_c1) { + CBMFileResult *r = extract("#if A\n" + "int pick(int a) { return a; }\n" + "#else\n" + "int pick(int a) {\n" + " return a + 1;\n" + "}\n" + "#endif\n" + "#ifdef B\n" + "#define LIMIT 1\n" + "#else\n" + "#define LIMIT 2\n" + "#endif\n" + "int once(void) { return 0; }\n", + CBM_LANG_C, "p", "v.c"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Function", "pick"), 2); + const CBMDefinition *kept = c1_def_at(r, "Function", "pick", 4); + const CBMDefinition *other = c1_def_at(r, "Function", "pick", 2); + ASSERT_NOT_NULL(kept); + ASSERT_NOT_NULL(other); + ASSERT_NOT_NULL(kept->variants); + ASSERT_STR_EQ(kept->variants, "[{\"start\":2,\"end\":2},{\"start\":4,\"end\":6}]"); + ASSERT_NULL(other->variants); /* one carrier per group: the node the graph keeps */ + + const CBMDefinition *lim_a = c1_def_at(r, "Macro", "LIMIT", 9); + const CBMDefinition *lim_b = c1_def_at(r, "Macro", "LIMIT", 11); + ASSERT_NOT_NULL(lim_a); + ASSERT_NOT_NULL(lim_b); + char want[128]; + snprintf(want, sizeof(want), "[{\"start\":%u,\"end\":%u},{\"start\":%u,\"end\":%u}]", + lim_a->start_line, lim_a->end_line, lim_b->start_line, lim_b->end_line); + ASSERT_NOT_NULL(lim_b->variants); + ASSERT_STR_EQ(lim_b->variants, want); + ASSERT_NULL(lim_a->variants); + + ASSERT_NULL(c1_def(r, "Function", "once")->variants); + cbm_free_result(r); + PASS(); +} + +/* `variants` describes conditional compilation, so only the C-preprocessor languages + * get it (cbm_is_c_preprocessor_lang, the six of the test above). A repeated QN + * elsewhere is another matter: a JSON file repeats one key name thousands of times + * (redis src/commands: 1,618 nodes, one of them with a 75-span list), a Python module + * may rebind a function. Both fixtures DO repeat a (QN, label) on DIFFERENT lines + * (two defs on one span count as one and would get no list anyway), which is asserted + * so the guard cannot go inert. */ +TEST(extract_variants_only_in_c_preprocessor_languages_c1) { + static const struct { + CBMLanguage lang; + const char *file; + const char *src; + } cases[] = { + {CBM_LANG_JSON, "cmd.json", "{\n \"a\": {\"name\": 1},\n \"b\": {\"name\": 2}\n}\n"}, + {CBM_LANG_PYTHON, "mod.py", + "def pick(a):\n return a\n\n\ndef pick(a):\n return a + 1\n"}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = extract(cases[i].src, cases[i].lang, "p", cases[i].file); + ASSERT_NOT_NULL(r); + int listed = 0; + bool distinct_lines = false; + for (int d = 0; d < r->defs.count; d++) { + const CBMDefinition *a = &r->defs.items[d]; + listed += a->variants != NULL; + for (int e = 0; a->qualified_name && a->label && e < d; e++) { + const CBMDefinition *b = &r->defs.items[e]; + distinct_lines = + distinct_lines || + (b->qualified_name && b->label && + strcmp(a->qualified_name, b->qualified_name) == 0 && + strcmp(a->label, b->label) == 0 && a->start_line != b->start_line); + } + } + if (listed != 0 || !distinct_lines) { + fprintf(stderr, + " [c1-guard] %s: defs with variants=%d, repeated on distinct lines=%d\n", + cases[i].file, listed, distinct_lines); + } + ASSERT_TRUE(distinct_lines); + ASSERT_EQ(listed, 0); + cbm_free_result(r); + } + PASS(); +} + /* #1989 review (P1): an over-long candidate-shaped identifier (>= 96 chars) * arriving AFTER a stored candidate must be rejected BEFORE the dedup * compares — the pre-fix dedup ran strncmp(out[k], src, len) with @@ -9566,6 +10301,22 @@ SUITE(extraction) { RUN_TEST(extract_cpp_export_macro_inline_method_recovery_issue1989); RUN_TEST(extract_cpp_export_macro_negative_control_ordinary_caps_issue1989); RUN_TEST(extract_c_export_macro_recovery_issue1989); + RUN_TEST(extract_cpp_macro_prefixed_function_names_c1); + RUN_TEST(extract_cpp_macro_prefixed_lsp_caller_c1); + RUN_TEST(extract_c_macro_prefixed_function_name_c1); + RUN_TEST(extract_c_tag_reference_is_not_a_definition_c1); + RUN_TEST(extract_c_struct_head_in_error_region_recovered_c1); + RUN_TEST(extract_c_keyword_is_never_a_function_name_c1); + RUN_TEST(extract_c_function_lost_to_split_branches_restored_c1); + RUN_TEST(extract_c_typedef_names_are_definitions_c1); + RUN_TEST(extract_c_macro_qn_is_fenced_c1); + RUN_TEST(extract_c_enumerators_are_flat_c1); + RUN_TEST(extract_cpp_enum_scoping_c1); + RUN_TEST(extract_c_enum_inside_struct_is_file_scope_c1); + RUN_TEST(extract_cpp_enum_inside_class_keeps_class_scope_c1); + RUN_TEST(extract_c_enum_macro_wrapped_list_c1); + RUN_TEST(extract_c_variants_lists_same_file_duplicates_c1); + RUN_TEST(extract_variants_only_in_c_preprocessor_languages_c1); RUN_TEST(extract_cpp_export_macro_overlong_candidate_safe_issue1989); RUN_TEST(extract_cpp_export_macro_comment_string_budget_issue1989); RUN_TEST(extract_cpp_export_macro_candidate_cap_issue1989); diff --git a/tests/test_graph_buffer.c b/tests/test_graph_buffer.c index 8a32d5a34..dd257f187 100644 --- a/tests/test_graph_buffer.c +++ b/tests/test_graph_buffer.c @@ -582,6 +582,71 @@ TEST(gbuf_upsert_same_qn_updates_all_fields) { PASS(); } +/* PR C1: when a Macro and a definition share a QN, the definition owns it whichever + * line comes first; before, the later line won. C-preprocessor macros no longer get + * here (their QN ends in "#macro" and cannot equal a definition's). Chialisp does: + * `defmacro` / `defmac` defs are labelled Macro under a plain QN, so a macro and a + * `defun` of one name collide in one file or in a same-stem .clsp / .clib pair. */ +TEST(gbuf_upsert_definition_beats_same_qn_macro_c1) { + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); + /* macro AFTER the function */ + cbm_gbuf_upsert_node(gb, "Function", "assert", "p.cond.assert", "cond.clsp", 742, 779, "{}"); + cbm_gbuf_upsert_node(gb, "Macro", "assert", "p.cond.assert", "cond.clsp", 781, 782, "{}"); + /* macro BEFORE the definition */ + cbm_gbuf_upsert_node(gb, "Macro", "curry", "p.util.curry", "util.clib", 464, 465, "{}"); + cbm_gbuf_upsert_node(gb, "Function", "curry", "p.util.curry", "util.clib", 3193, 3209, "{}"); + /* a macro in the smaller path (.clib sorts before .clsp) still loses */ + cbm_gbuf_upsert_node(gb, "Constant", "limit", "p.x.limit", "x.clsp", 653, 653, "{}"); + cbm_gbuf_upsert_node(gb, "Macro", "limit", "p.x.limit", "x.clib", 9, 10, "{}"); + /* two macros: the classic contract (later line wins) is unchanged */ + cbm_gbuf_upsert_node(gb, "Macro", "M", "p.m.M", "m.clib", 1, 2, "{}"); + cbm_gbuf_upsert_node(gb, "Macro", "M", "p.m.M", "m.clib", 9, 10, "{}"); + + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(gb, "p.cond.assert"); + ASSERT_NOT_NULL(n); + ASSERT_STR_EQ(n->label, "Function"); + ASSERT_EQ(n->start_line, 742); + n = cbm_gbuf_find_by_qn(gb, "p.util.curry"); + ASSERT_NOT_NULL(n); + ASSERT_STR_EQ(n->label, "Function"); + ASSERT_EQ(n->start_line, 3193); + n = cbm_gbuf_find_by_qn(gb, "p.x.limit"); + ASSERT_NOT_NULL(n); + ASSERT_STR_EQ(n->label, "Constant"); + ASSERT_STR_EQ(n->file_path, "x.clsp"); + n = cbm_gbuf_find_by_qn(gb, "p.m.M"); + ASSERT_NOT_NULL(n); + ASSERT_EQ(n->start_line, 9); + /* the label index follows the survivor: only the macro-vs-macro QN keeps one */ + const cbm_gbuf_node_t **macros = NULL; + int macro_count = -1; + ASSERT_EQ(cbm_gbuf_find_by_label(gb, "Macro", ¯os, ¯o_count), 0); + ASSERT_EQ(macro_count, 1); + cbm_gbuf_free(gb); + PASS(); +} + +/* The parallel merge (worker gbuf -> main gbuf) applies the same rule in both + * directions, so the survivor never depends on worker merge order. */ +TEST(gbuf_merge_definition_beats_same_qn_macro_c1) { + for (int order = 0; order < 2; order++) { + cbm_gbuf_t *dst = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_t *src = cbm_gbuf_new("test", "/tmp"); + cbm_gbuf_t *with_fn = order == 0 ? dst : src; + cbm_gbuf_t *with_macro = order == 0 ? src : dst; + cbm_gbuf_upsert_node(with_fn, "Function", "f", "p.a.f", "a.clsp", 10, 20, "{}"); + cbm_gbuf_upsert_node(with_macro, "Macro", "f", "p.a.f", "a.clsp", 30, 31, "{}"); + ASSERT_EQ(cbm_gbuf_merge(dst, src), 0); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(dst, "p.a.f"); + ASSERT_NOT_NULL(n); + ASSERT_STR_EQ(n->label, "Function"); + ASSERT_EQ(n->start_line, 10); + cbm_gbuf_free(dst); + cbm_gbuf_free(src); + } + PASS(); +} + TEST(gbuf_upsert_long_qn) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp"); @@ -1250,6 +1315,8 @@ SUITE(graph_buffer) { RUN_TEST(gbuf_upsert_null_qn); RUN_TEST(gbuf_upsert_empty_qn); RUN_TEST(gbuf_upsert_same_qn_updates_all_fields); + RUN_TEST(gbuf_upsert_definition_beats_same_qn_macro_c1); + RUN_TEST(gbuf_merge_definition_beats_same_qn_macro_c1); RUN_TEST(gbuf_upsert_long_qn); RUN_TEST(gbuf_find_by_qn_missing); RUN_TEST(gbuf_find_by_id_missing); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index c5705ddcf..2dd2621db 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -6325,6 +6325,50 @@ TEST(tool_trace_call_path_ambiguous) { PASS(); } +/* Tie rule for the C-macro namespace (PR C1): a typedef and its rename macro + * (`typedef struct state_s\n state_t;` + `#define state_t NS(state_t)`) are + * two nodes with one name, `.state_t` and `.state_t#macro`, and + * here the same line span. The name resolves to the definition: without the + * macro ranking below it, the two tie and trace_path answers "ambiguous". */ +TEST(tool_trace_path_definition_beats_c_macro_c1) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "tie-proj"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/tie"); + cbm_node_t type = {.project = proj, + .label = "Type", + .name = "state_t", + .qualified_name = "tie-proj.xxhash.state_t", + .file_path = "xxhash.h", + .start_line = 653, + .end_line = 654}; + cbm_node_t macro = {.project = proj, + .label = "Macro", + .name = "state_t", + .qualified_name = "tie-proj.xxhash.state_t#macro", + .file_path = "xxhash.h", + .start_line = 429, + .end_line = 430}; /* equal span: a tie without the rule */ + ASSERT_GT(cbm_store_upsert_node(st, &type), 0); + ASSERT_GT(cbm_store_upsert_node(st, ¯o), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":62,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_call_path\"," + "\"arguments\":{\"function_name\":\"state_t\",\"project\":\"tie-proj\"}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "ambiguous")); + ASSERT_NULL(strstr(inner, "suggestions")); + ASSERT_NULL(strstr(inner, "function not found")); + free(inner); + free(resp); + cbm_mcp_server_free(srv); + PASS(); +} + /* Multi-seed union hop semantics: bfs_union_same_name deduped visited nodes * keep-FIRST-seen, so a node reached at hop 2 from the first seed kept hop 2 * even when the second seed reaches it at hop 1. hop feeds risk_labels and @@ -16818,6 +16862,79 @@ TEST(snippet_unique_short_name) { PASS(); } +/* ── C-macro namespace (PR C1) ────────────────────────────────── */ + +/* A C macro's QN ends in "#macro", which no caller spells. get_code_snippet + * still returns the macro for its short name, for a dotted suffix and for the + * QN it had before the fence (`.`); when a definition owns that + * QN or that name, the definition is the answer. */ +TEST(snippet_c_macro_namespace_c1) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + + cbm_node_t only_macro = {.project = "test-project", + .label = "Macro", + .name = "MAX_LEN", + .qualified_name = "test-project.lib.cfg.MAX_LEN#macro", + .file_path = "main.go", + .start_line = 7, + .end_line = 8}; + cbm_node_t type = {.project = "test-project", + .label = "Type", + .name = "state_t", + .qualified_name = "test-project.lib.xx.state_t", + .file_path = "main.go", + .start_line = 3, + .end_line = 3}; + cbm_node_t shadowed_macro = {.project = "test-project", + .label = "Macro", + .name = "state_t", + .qualified_name = "test-project.lib.xx.state_t#macro", + .file_path = "main.go", + .start_line = 11, + .end_line = 12}; + ASSERT_GT(cbm_store_upsert_node(st, &only_macro), 0); + ASSERT_GT(cbm_store_upsert_node(st, &type), 0); + ASSERT_GT(cbm_store_upsert_node(st, &shadowed_macro), 0); + + static const char *const macro_inputs[] = { + "{\"qualified_name\":\"MAX_LEN\",\"project\":\"test-project\"}", + "{\"qualified_name\":\"cfg.MAX_LEN\",\"project\":\"test-project\"}", + "{\"qualified_name\":\"test-project.lib.cfg.MAX_LEN\",\"project\":\"test-project\"}", + "{\"qualified_name\":\"test-project.lib.cfg.MAX_LEN#macro\",\"project\":\"test-project\"}"}; + for (size_t i = 0; i < sizeof(macro_inputs) / sizeof(macro_inputs[0]); i++) { + char *resp = call_snippet(srv, macro_inputs[i]); + ASSERT_NOT_NULL(resp); + if (!strstr(resp, "\"qualified_name\":\"test-project.lib.cfg.MAX_LEN#macro\"")) { + fprintf(stderr, " [c1-snippet] %s -> %.300s\n", macro_inputs[i], resp); + } + ASSERT_NOT_NULL(strstr(resp, "\"qualified_name\":\"test-project.lib.cfg.MAX_LEN#macro\"")); + ASSERT_NOT_NULL(strstr(resp, "\"label\":\"Macro\"")); + free(resp); + } + + static const char *const definition_inputs[] = { + "{\"qualified_name\":\"test-project.lib.xx.state_t\",\"project\":\"test-project\"}", + "{\"qualified_name\":\"xx.state_t\",\"project\":\"test-project\"}", + "{\"qualified_name\":\"state_t\",\"project\":\"test-project\"}"}; + for (size_t i = 0; i < sizeof(definition_inputs) / sizeof(definition_inputs[0]); i++) { + char *resp = call_snippet(srv, definition_inputs[i]); + ASSERT_NOT_NULL(resp); + if (!strstr(resp, "\"label\":\"Type\"")) { + fprintf(stderr, " [c1-snippet] %s -> %.300s\n", definition_inputs[i], resp); + } + ASSERT_NOT_NULL(strstr(resp, "\"qualified_name\":\"test-project.lib.xx.state_t\"")); + ASSERT_NOT_NULL(strstr(resp, "\"label\":\"Type\"")); + free(resp); + } + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + /* ── TestSnippet_NameTier ─────────────────────────────────────── */ TEST(snippet_name_tier) { @@ -21101,6 +21218,7 @@ SUITE(mcp) { RUN_TEST(tool_call_invalid_project_name_leaves_no_corrupt_litter_issue1425); RUN_TEST(tool_trace_missing_function_name); RUN_TEST(tool_trace_call_path_ambiguous); + RUN_TEST(tool_trace_path_definition_beats_c_macro_c1); RUN_TEST(tool_trace_union_records_min_hop_across_seeds); RUN_TEST(tool_trace_pagination_exactly_once); RUN_TEST(tool_trace_paging_filters_before_window_and_hashes_effective_args); @@ -21275,6 +21393,7 @@ SUITE(mcp) { RUN_TEST(snippet_exact_qn); RUN_TEST(snippet_qn_suffix); RUN_TEST(snippet_unique_short_name); + RUN_TEST(snippet_c_macro_namespace_c1); RUN_TEST(snippet_name_tier); RUN_TEST(snippet_ambiguous_short_name); RUN_TEST(snippet_not_found); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9a4c461c6..34f0ba8c2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8576,6 +8576,581 @@ TEST(pipeline_python_project) { PASS(); } +/* Edges of `edge_type` from the node `.` to `.` + * (QNs, not names: a C macro and the definition it shadows share their name). */ +static int c1_edge_count_qn(cbm_store_t *s, const char *project, const char *edge_type, + const char *source_tail, const char *target_tail) { + char source_qn[512]; + char target_qn[512]; + snprintf(source_qn, sizeof(source_qn), "%s.%s", project, source_tail); + snprintf(target_qn, sizeof(target_qn), "%s.%s", project, target_tail); + cbm_node_t source = {0}; + cbm_node_t target = {0}; + int matches = -1; + if (cbm_store_find_node_by_qn(s, project, source_qn, &source) == CBM_STORE_OK && + cbm_store_find_node_by_qn(s, project, target_qn, &target) == CBM_STORE_OK) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + matches = 0; + if (cbm_store_find_edges_by_source_type(s, source.id, edge_type, &edges, &edge_count) == + CBM_STORE_OK) { + for (int i = 0; i < edge_count; i++) { + matches += edges[i].target_id == target.id; + } + cbm_store_free_edges(edges, edge_count); + } + } + cbm_node_free_fields(&source); + cbm_node_free_fields(&target); + return matches; +} + +/* Label of the node `.`, copied into `out`; "" when there is none. */ +static const char *c1_label_of(cbm_store_t *s, const char *project, const char *qn_tail, char *out, + size_t out_size) { + char qn[512]; + snprintf(qn, sizeof(qn), "%s.%s", project, qn_tail); + cbm_node_t node = {0}; + out[0] = '\0'; + if (cbm_store_find_node_by_qn(s, project, qn, &node) == CBM_STORE_OK && node.label) { + snprintf(out, out_size, "%s", node.label); + } + cbm_node_free_fields(&node); + return out; +} + +/* Do the properties of the node `.` contain `needle`? */ +static bool c1_props_contain(cbm_store_t *s, const char *project, const char *qn_tail, + const char *needle) { + char qn[512]; + snprintf(qn, sizeof(qn), "%s.%s", project, qn_tail); + cbm_node_t node = {0}; + bool found = cbm_store_find_node_by_qn(s, project, qn, &node) == CBM_STORE_OK && + node.properties_json && strstr(node.properties_json, needle) != NULL; + cbm_node_free_fields(&node); + return found; +} + +/* PR C1, end to end: C keeps one graph node per QN, so the right entity must win. + * A header and its .c share the module QN (proj.s): `struct S s_global;` in s.c + * minted a Class for the REFERENCE and "smallest path wins" handed it the node (redis: + * struct redisServer pointed at server.c:85). A function and its #else macro + * stand-in shared a QN, and the later macro line won (curl url_match_proxy_use, 41 + * functions): the macro now has a QN of its own (`...#macro`), so both are nodes. + * The typedef alias had no node at all. */ +TEST(pipeline_c_definitions_own_their_qn_c1) { + const char *files[] = {"s.h", "s.c"}; + const char *contents[] = {"struct S {\n" + " int v;\n" + "};\n" + "typedef struct S S_t;\n", + + "#include \"s.h\"\n" + "struct S s_global;\n" + "#ifndef DISABLE_PROXY\n" + "static int match_proxy(int a)\n" + "{\n" + " return a + 1;\n" + "}\n" + "#else\n" + "#define match_proxy(a) 1\n" + "#endif\n" + "int use(void) { return match_proxy(s_global.v); }\n"}; + if (setup_lang_repo(files, contents, 2) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + const char *proj = cbm_pipeline_project_name(p); + + cbm_node_t *nodes = NULL; + int n = 0; + cbm_store_find_nodes_by_name(s, proj, "S", &nodes, &n); + ASSERT_EQ(n, 1); + ASSERT_STR_EQ(nodes[0].label, "Class"); + ASSERT_STR_EQ(nodes[0].file_path, "s.h"); + ASSERT_EQ(nodes[0].start_line, 1); + cbm_store_free_nodes(nodes, n); + + /* the function and its #else macro stand-in: two nodes, the plain QN is the + * function's */ + cbm_store_find_nodes_by_name(s, proj, "match_proxy", &nodes, &n); + ASSERT_EQ(n, 2); + cbm_store_free_nodes(nodes, n); + char qn[512]; + cbm_node_t fn = {0}; + snprintf(qn, sizeof(qn), "%s.s.match_proxy", proj); + ASSERT_EQ(cbm_store_find_node_by_qn(s, proj, qn, &fn), CBM_STORE_OK); + ASSERT_STR_EQ(fn.label, "Function"); + ASSERT_EQ(fn.start_line, 4); + cbm_node_free_fields(&fn); + cbm_node_t mac = {0}; + snprintf(qn, sizeof(qn), "%s.s.match_proxy#macro", proj); + ASSERT_EQ(cbm_store_find_node_by_qn(s, proj, qn, &mac), CBM_STORE_OK); + ASSERT_STR_EQ(mac.label, "Macro"); + ASSERT_STR_EQ(mac.name, "match_proxy"); + ASSERT_EQ(mac.start_line, 9); + cbm_node_free_fields(&mac); + + cbm_store_find_nodes_by_name(s, proj, "S_t", &nodes, &n); + ASSERT_EQ(n, 1); + ASSERT_STR_EQ(nodes[0].label, "Type"); + cbm_store_free_nodes(nodes, n); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + PASS(); +} + +/* The tie rule of the macro namespace: a name visible both as a definition and as a + * macro resolves to the DEFINITION; the macro is the target only when no definition + * of that name is visible. `pick_fn` is a function in one #if branch and a macro in + * the other: before, one node held the QN and it was the macro (the later line), so + * the call landed on a Macro. `renamed_fn` is declared by a prototype whose name a + * rename macro replaces (jemalloc smallocx): no definition exists in the repo, the + * macro is the only node, and the call keeps pointing at it. */ +TEST(pipeline_c_call_targets_definition_before_macro_c1) { + const char *files[] = {"tie.c"}; + const char *contents[] = {"#define renamed_fn NS_renamed_fn\n" + "extern int renamed_fn(int v);\n" + "#ifdef USE_REAL\n" + "static int pick_fn(int v) { return v; }\n" + "#else\n" + "#define pick_fn(v) (v)\n" + "#endif\n" + "int caller(int b) {\n" + " return pick_fn(b) + renamed_fn(b);\n" + "}\n"}; + if (setup_lang_repo(files, contents, 1) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char proj[256]; + snprintf(proj, sizeof(proj), "%s", cbm_pipeline_project_name(p)); + + char label[64]; + ASSERT_STR_EQ(c1_label_of(s, proj, "tie.pick_fn", label, sizeof(label)), "Function"); + ASSERT_STR_EQ(c1_label_of(s, proj, "tie.pick_fn#macro", label, sizeof(label)), "Macro"); + ASSERT_STR_EQ(c1_label_of(s, proj, "tie.renamed_fn#macro", label, sizeof(label)), "Macro"); + ASSERT_STR_EQ(c1_label_of(s, proj, "tie.renamed_fn", label, sizeof(label)), ""); + int to_definition = c1_edge_count_qn(s, proj, "CALLS", "tie.caller", "tie.pick_fn"); + int to_shadowed_macro = c1_edge_count_qn(s, proj, "CALLS", "tie.caller", "tie.pick_fn#macro"); + int to_only_macro = c1_edge_count_qn(s, proj, "CALLS", "tie.caller", "tie.renamed_fn#macro"); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + + ASSERT_EQ(to_definition, 1); + ASSERT_EQ(to_shadowed_macro, 0); + ASSERT_EQ(to_only_macro, 1); + PASS(); +} + +/* A flattened enumerator is still what `E::A` names: the reference resolves by its + * leaf, so `Color::RED`, a namespace-qualified and a class-qualified spelling and the + * bare `GREEN` all reach the node `.`; a scoped enum's `Mode::FAST` + * reaches `.Mode.FAST`. */ +TEST(pipeline_cpp_enum_reference_reaches_flat_enumerator_c1) { + const char *files[] = {"shapes.hpp", "use.cpp"}; + const char *contents[] = {"enum Color { RED, GREEN };\n" + "namespace gfx {\n" + "enum Blend { ADD, MULTIPLY };\n" + "class Brush {\n" + "public:\n" + " enum Shape { ROUND, SQUARE };\n" + "};\n" + "}\n" + "enum class Mode { FAST, SLOW };\n", + + "#include \"shapes.hpp\"\n" + "int pick(int v) {\n" + " if (v == Color::RED) return 1;\n" + " if (v == gfx::Blend::ADD) return 2;\n" + " if (v == gfx::Brush::ROUND) return 3;\n" + " if (v == static_cast(Mode::FAST)) return 4;\n" + " if (v == GREEN) return 5;\n" + " return 0;\n" + "}\n"}; + if (setup_lang_repo(files, contents, 2) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char proj[256]; + snprintf(proj, sizeof(proj), "%s", cbm_pipeline_project_name(p)); + + static const char *const targets[] = {"shapes.RED", "shapes.gfx.ADD", "shapes.gfx.Brush.ROUND", + "shapes.Mode.FAST", "shapes.GREEN"}; + int usage[5]; + for (int i = 0; i < 5; i++) { + usage[i] = c1_edge_count_qn(s, proj, "USAGE", "use.pick", targets[i]); + } + char nested[64]; + c1_label_of(s, proj, "shapes.Color.RED", nested, sizeof(nested)); + + /* "the constants of enum Color" still enumerates: parent_class is the membership */ + char parent_marker[512]; + snprintf(parent_marker, sizeof(parent_marker), "\"parent_class\":\"%s.shapes.Color\"", proj); + cbm_node_t *vars = NULL; + int var_count = 0; + int color_members = 0; + cbm_store_find_nodes_by_label(s, proj, "Variable", &vars, &var_count); + for (int i = 0; i < var_count; i++) { + color_members += + vars[i].properties_json && strstr(vars[i].properties_json, parent_marker) != NULL; + } + cbm_store_free_nodes(vars, var_count); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + + for (int i = 0; i < 5; i++) { + if (usage[i] != 1) { + fprintf(stderr, " [c1] USAGE use.pick -> %s: %d\n", targets[i], usage[i]); + } + ASSERT_EQ(usage[i], 1); + } + ASSERT_STR_EQ(nested, ""); + ASSERT_EQ(color_members, 2); + PASS(); +} + +/* C, an enum declared inside a struct (curl lib/cf-h1-proxy.c `enum keeponval {...} + * keepon;`): its enumerators are file-scope names, so their nodes are `.` + * and the unqualified references in the code reach them. The struct is no QN segment, + * for the named enum and the anonymous one alike. */ +TEST(pipeline_c_enum_inside_struct_reference_resolves_c1) { + const char *files[] = {"link.c"}; + const char *contents[] = {"struct conn {\n" + " enum state { ST_IDLE, ST_BUSY } st;\n" + " enum { KIND_A, KIND_B } kind;\n" + " int fd;\n" + "};\n" + "int busy(struct conn *c) {\n" + " return c->st == ST_BUSY || c->kind == KIND_B;\n" + "}\n"}; + if (setup_lang_repo(files, contents, 1) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char proj[256]; + snprintf(proj, sizeof(proj), "%s", cbm_pipeline_project_name(p)); + + int to_named = c1_edge_count_qn(s, proj, "USAGE", "link.busy", "link.ST_BUSY"); + int to_anonymous = c1_edge_count_qn(s, proj, "USAGE", "link.busy", "link.KIND_B"); + char in_struct[64]; + c1_label_of(s, proj, "link.conn.ST_BUSY", in_struct, sizeof(in_struct)); + char parent_marker[512]; + snprintf(parent_marker, sizeof(parent_marker), "\"parent_class\":\"%s.link.conn.state\"", proj); + bool named_keeps_parent = c1_props_contain(s, proj, "link.ST_BUSY", parent_marker); + bool anonymous_has_parent = c1_props_contain(s, proj, "link.KIND_B", "\"parent_class\""); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + + ASSERT_EQ(to_named, 1); + ASSERT_EQ(to_anonymous, 1); + ASSERT_STR_EQ(in_struct, ""); + ASSERT_TRUE(named_keeps_parent); + ASSERT_FALSE(anonymous_has_parent); + PASS(); +} + +typedef struct { + int rc; + bool opened; + bool pick_lists_both; /* .v.pick carries the two spans, in order */ + bool limit_has_list; /* .v.LIMIT#macro carries a list */ + bool once_has_list; /* a name defined once must not */ + int other_lang_nodes; /* nodes of cmd.json */ + int other_lang_lists; /* ... that carry a list (must be 0) */ +} c1_variants_obs_t; + +static c1_variants_obs_t c1_observe_variants(const char *repo, const char *db_name) { + c1_variants_obs_t obs = {0}; + char db[512]; + snprintf(db, sizeof(db), "%s/%s", repo, db_name); + cbm_pipeline_t *p = cbm_pipeline_new(repo, db, CBM_MODE_FULL); + if (!p) { + obs.rc = -1; + return obs; + } + obs.rc = cbm_pipeline_run(p); + char proj[256]; + snprintf(proj, sizeof(proj), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + cbm_store_t *s = cbm_store_open_path(db); + if (!s) { + return obs; + } + obs.opened = true; + obs.pick_lists_both = c1_props_contain( + s, proj, "v.pick", "\"variants\":[{\"start\":2,\"end\":2},{\"start\":4,\"end\":6}]"); + obs.limit_has_list = c1_props_contain(s, proj, "v.LIMIT#macro", "\"variants\":[{\"start\":9,"); + obs.once_has_list = c1_props_contain(s, proj, "v.once", "\"variants\""); + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file_overlap(s, proj, "cmd.json", 1, 1000, &nodes, &count) == + CBM_STORE_OK) { + obs.other_lang_nodes = count; + for (int i = 0; i < count; i++) { + obs.other_lang_lists += nodes[i].properties_json && + strstr(nodes[i].properties_json, "\"variants\"") != NULL; + } + cbm_store_free_nodes(nodes, count); + } + cbm_store_close(s); + return obs; +} + +/* `variants` reaches the node on BOTH execution paths. The property is computed per + * file at extraction; the sequential pass (pass_definitions.c) and the parallel one + * (pass_parallel.c) each serialize it, and the properties buffer is sized for the + * list, so a path that forgot either would publish a node without it. One fixture, + * two runs: CBM_INDEX_SINGLE_THREAD forces the sequential path, the 55 fillers plus + * CBM_WORKERS select the parallel one. A JSON file with a repeated key rides along: + * no language outside the C preprocessor ones gets the property. */ +TEST(pipeline_c_variants_on_sequential_and_parallel_paths_c1) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_c1_variants_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/v.c", tmp); + int write_rc = th_write_file(path, "#if A\n" + "int pick(int a) { return a; }\n" + "#else\n" + "int pick(int a) {\n" + " return a + 1;\n" + "}\n" + "#endif\n" + "#ifdef B\n" + "#define LIMIT 1\n" + "#else\n" + "#define LIMIT 2\n" + "#endif\n" + "int once(void) { return 0; }\n"); + snprintf(path, sizeof(path), "%s/cmd.json", tmp); + /* the repeated key on two lines: one span would never be listed, gate or no gate */ + write_rc |= th_write_file(path, "{\n \"a\": {\"name\": 1},\n \"b\": {\"name\": 2}\n}\n"); + for (int i = 0; i < 55; i++) { + char source[96]; + snprintf(path, sizeof(path), "%s/pad_%02d.c", tmp, i); + snprintf(source, sizeof(source), "int c1_pad_%02d(void) { return %d; }\n", i, i); + write_rc |= th_write_file(path, source); + } + if (write_rc != 0) { + th_rmtree(tmp); + FAIL("failed to write the variants fixture"); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + c1_variants_obs_t sequential = c1_observe_variants(tmp, "variants-sequential.db"); + + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + cbm_setenv("CBM_WORKERS", "4", 1); + c1_variants_obs_t parallel = c1_observe_variants(tmp, "variants-parallel.db"); + + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (saved_single) { + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + free(saved_single); + } else { + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + } + th_rmtree(tmp); + + const c1_variants_obs_t *runs[] = {&sequential, ¶llel}; + for (int i = 0; i < 2; i++) { + if (!runs[i]->pick_lists_both || !runs[i]->limit_has_list || runs[i]->once_has_list || + runs[i]->other_lang_lists != 0) { + fprintf( + stderr, + " [c1-variants] path=%s pick=%d limit=%d once=%d json_nodes=%d json_lists=%d\n", + i == 0 ? "sequential" : "parallel", runs[i]->pick_lists_both, + runs[i]->limit_has_list, runs[i]->once_has_list, runs[i]->other_lang_nodes, + runs[i]->other_lang_lists); + } + } + /* the property on each path first, the language limit second: one concern must + * not hide the other */ + for (int i = 0; i < 2; i++) { + ASSERT_EQ(runs[i]->rc, 0); + ASSERT_TRUE(runs[i]->opened); + ASSERT_TRUE(runs[i]->pick_lists_both); + ASSERT_TRUE(runs[i]->limit_has_list); + ASSERT_FALSE(runs[i]->once_has_list); + } + for (int i = 0; i < 2; i++) { + ASSERT_GT(runs[i]->other_lang_nodes, 0); + ASSERT_EQ(runs[i]->other_lang_lists, 0); + } + PASS(); +} + +/* C++ overloads share a QN (the signature is no part of it), so two of them in one + * file are the same shape as two #if branches: one node -- the last by start line -- + * and `variants` lists both spans. */ +TEST(pipeline_cpp_overloads_are_listed_as_variants_c1) { + const char *files[] = {"ov.cpp"}; + const char *contents[] = {"int scale(int v) { return v * 2; }\n" + "double scale(double v) {\n" + " return v * 2.0;\n" + "}\n" + "int once(int v) { return scale(v); }\n"}; + if (setup_lang_repo(files, contents, 1) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + char proj[256]; + snprintf(proj, sizeof(proj), "%s", cbm_pipeline_project_name(p)); + + cbm_node_t *nodes = NULL; + int n = 0; + cbm_store_find_nodes_by_name(s, proj, "scale", &nodes, &n); + int scale_nodes = n; + int kept_start = n > 0 ? nodes[0].start_line : -1; + cbm_store_free_nodes(nodes, n); + bool lists_both = c1_props_contain( + s, proj, "ov.scale", "\"variants\":[{\"start\":1,\"end\":1},{\"start\":2,\"end\":4}]"); + bool once_has_list = c1_props_contain(s, proj, "ov.once", "\"variants\""); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + + ASSERT_EQ(scale_nodes, 1); + ASSERT_EQ(kept_start, 2); + ASSERT_TRUE(lists_both); + ASSERT_FALSE(once_has_list); + PASS(); +} + +/* CBM_SEMANTIC_INDEX_VERSION 4: the C-family node identities changed (macro QNs end + * in "#macro", unscoped enumerators are flat, typedef names are nodes). An index + * written at version 3 holds the old QNs for every file that did not change, and an + * unchanged repository is otherwise a no-op, so the version is what makes the new + * binary rebuild it. The stored index is put back to the version-3 state by hand + * (metadata and the old macro QN); the run after that must replace it in full. */ +TEST(pipeline_semantic_version_3_index_is_rebuilt_in_full_c1) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_c1_version_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "cfg.c", "#define LIMIT 10\nint limit(void) { return LIMIT; }\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/index.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(first)); + cbm_pipeline_free(first); + + /* Put the published index back to what version 3 wrote. */ + cbm_store_t *store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + char label[64]; + bool fenced_before = + strcmp(c1_label_of(store, project, "cfg.LIMIT#macro", label, sizeof(label)), "Macro") == 0; + cbm_coverage_row_t *coverage_rows = NULL; + int coverage_count = 0; + ASSERT_EQ(cbm_store_coverage_get(store, project, &coverage_rows, &coverage_count), + CBM_STORE_OK); + cbm_coverage_meta_t meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(store, project, &meta), CBM_STORE_OK); + cbm_coverage_meta_t old_meta = meta; + old_meta.coverage_version = 3; + ASSERT_EQ( + cbm_store_coverage_replace_ex(store, project, coverage_rows, coverage_count, &old_meta), + CBM_STORE_OK); + cbm_store_free_coverage(coverage_rows, coverage_count); + cbm_store_coverage_meta_clear(&meta); + ASSERT_EQ(cbm_store_exec(store, "UPDATE nodes SET qualified_name = " + "substr(qualified_name, 1, length(qualified_name) - 6) " + "WHERE label = 'Macro' AND qualified_name LIKE '%#macro';"), + CBM_STORE_OK); + bool plain_after_downgrade = + strcmp(c1_label_of(store, project, "cfg.LIMIT", label, sizeof(label)), "Macro") == 0; + cbm_store_close(store); + + /* Nothing in the repository changed: only the version says the index is stale. */ + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *upgrade = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(upgrade); + int upgrade_rc = cbm_pipeline_run(upgrade); + cbm_incremental_route_t upgrade_route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(upgrade); + + store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + bool fenced_after = + strcmp(c1_label_of(store, project, "cfg.LIMIT#macro", label, sizeof(label)), "Macro") == 0; + bool plain_after = + strcmp(c1_label_of(store, project, "cfg.LIMIT", label, sizeof(label)), "Macro") == 0; + cbm_coverage_meta_t new_meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(store, project, &new_meta), CBM_STORE_OK); + int stored_version = new_meta.coverage_version; + cbm_store_coverage_meta_clear(&new_meta); + cbm_store_close(store); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_TRUE(fenced_before); + ASSERT_TRUE(plain_after_downgrade); + ASSERT_EQ(upgrade_rc, 0); + ASSERT_EQ(upgrade_route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(stored_version, CBM_SEMANTIC_INDEX_VERSION); + ASSERT_TRUE(fenced_after); + ASSERT_FALSE(plain_after); + ASSERT_GTE(CBM_SEMANTIC_INDEX_VERSION, 4); + PASS(); +} + /* `#include ` from arch/x/bugs.c: two headers end with the * include path (include/linux/device.h, tools/virtio/linux/device.h) and each * declares `struct device`. The exact-file lookup returned the first matching @@ -16462,6 +17037,13 @@ SUITE(pipeline) { RUN_TEST(usages_kotlin_no_duplicate_calls); /* Language integration tests */ RUN_TEST(pipeline_python_project); + RUN_TEST(pipeline_c_definitions_own_their_qn_c1); + RUN_TEST(pipeline_c_call_targets_definition_before_macro_c1); + RUN_TEST(pipeline_cpp_enum_reference_reaches_flat_enumerator_c1); + RUN_TEST(pipeline_c_enum_inside_struct_reference_resolves_c1); + RUN_TEST(pipeline_c_variants_on_sequential_and_parallel_paths_c1); + RUN_TEST(pipeline_cpp_overloads_are_listed_as_variants_c1); + RUN_TEST(pipeline_semantic_version_3_index_is_rebuilt_in_full_c1); RUN_TEST(pipeline_header_include_target_is_independent_of_registration_order); RUN_TEST(pipeline_imports_multi_symbol_edges); RUN_TEST(pipeline_go_cross_package_call); diff --git a/tests/test_store_search.c b/tests/test_store_search.c index 9110c420a..4c1063bcc 100644 --- a/tests/test_store_search.c +++ b/tests/test_store_search.c @@ -8,6 +8,7 @@ #include "test_framework.h" #include "test_helpers.h" #include +#include "cbm.h" /* CBM_MACRO_QN_SUFFIX — the fence the FTS feed strips */ #include "sqlite3.h" /* vendored/sqlite3 — raw nodes_fts MATCH probes */ #include #include @@ -1797,6 +1798,66 @@ TEST(store_fts_rebuild_incremental_adds_only_nodes_above_watermark) { PASS(); } +/* A C macro's QN ends in "#macro" (CBM_MACRO_QN_SUFFIX) and `#` separates tokens, so + * indexing that QN as it is would give every macro node the word "macro" a second time + * (its label already says Macro). On redis that is 5,513 rows outscoring the two nodes + * that carry the word in their NAME: tre_expand_macro fell out of the 2,000-row BM25 + * candidate window and out of the result for the query `macro`. The qualified_name + * column therefore gets a Macro's QN without the fence, and only a Macro's. The + * wholesale rebuild and the delta-merge rebuild are one writer; both are run here. */ +TEST(store_fts_macro_qn_fence_adds_no_token_c1) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_store_upsert_project(s, "p", "/tmp/p"); + cbm_node_t macro = {.project = "p", + .label = "Macro", + .name = "LIMIT", + .qualified_name = "p.cfg.LIMIT" CBM_MACRO_QN_SUFFIX, + .file_path = "cfg.h"}; + cbm_node_t named = {.project = "p", + .label = "Function", + .name = "expandMacro", + .qualified_name = "p.cfg.expandMacro", + .file_path = "cfg.c"}; + /* not a Macro: a QN that merely ends like the fence keeps all its tokens */ + cbm_node_t other = {.project = "p", + .label = "Function", + .name = "odd", + .qualified_name = "p.cfg.odd#macro", + .file_path = "cfg.c"}; + ASSERT_TRUE(cbm_store_upsert_node(s, ¯o) > 0); + ASSERT_TRUE(cbm_store_upsert_node(s, &named) > 0); + ASSERT_TRUE(cbm_store_upsert_node(s, &other) > 0); + + ASSERT_EQ(cbm_store_fts_rebuild(s, NULL, 0), CBM_STORE_OK); + ASSERT_EQ(fts_match_count(s, "qualified_name:LIMIT"), 1); /* the QN stays searchable */ + ASSERT_EQ(fts_match_count(s, "label:Macro"), 1); + ASSERT_EQ(fts_match_count(s, "qualified_name:macro"), 1); /* `odd` alone */ + ASSERT_EQ(fts_match_count(s, "name:Macro"), 1); /* expandMacro, camel-split */ + + sqlite3_stmt *st = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), "SELECT COALESCE(MAX(id),0) FROM nodes", -1, + &st, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(st), SQLITE_ROW); + int64_t watermark = sqlite3_column_int64(st, 0); + sqlite3_finalize(st); + + cbm_node_t added = {.project = "p", + .label = "Macro", + .name = "DEPTH", + .qualified_name = "p.cfg.DEPTH" CBM_MACRO_QN_SUFFIX, + .file_path = "cfg.h"}; + ASSERT_TRUE(cbm_store_upsert_node(s, &added) > 0); + ASSERT_EQ(cbm_store_fts_rebuild(s, "p", watermark), CBM_STORE_OK); + ASSERT_EQ(fts_match_count(s, "qualified_name:DEPTH"), 1); + ASSERT_EQ(fts_match_count(s, "label:Macro"), 2); + ASSERT_EQ(fts_match_count(s, "qualified_name:macro"), 1); /* the delta path strips it too */ + + cbm_store_close(s); + PASS(); +} + SUITE(store_search) { RUN_TEST(store_search_by_label); RUN_TEST(store_search_by_name_pattern); @@ -1873,4 +1934,5 @@ SUITE(store_search) { RUN_TEST(store_fts_rebuild_survives_malformed_properties_json); RUN_TEST(store_fts_rebuild_tolerates_legacy_four_column_table); RUN_TEST(store_fts_rebuild_incremental_adds_only_nodes_above_watermark); + RUN_TEST(store_fts_macro_qn_fence_adds_no_token_c1); } From 5692dbe29982a344ee2d712957f75ece23a6557e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 20:56:41 +0200 Subject: [PATCH 11/34] ci: retry MSan apt updates after transient repository errors Signed-off-by: Martin Vogel --- test-infrastructure/Dockerfile.msan | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test-infrastructure/Dockerfile.msan b/test-infrastructure/Dockerfile.msan index ce9b616d0..144abf279 100644 --- a/test-infrastructure/Dockerfile.msan +++ b/test-infrastructure/Dockerfile.msan @@ -51,17 +51,19 @@ RUN printf '%s\n' \ # The key is fetched with `wget -O ` rather than `-qO- > `: under # retry a redirect is opened once for all attempts, so a partial write from a # failed attempt would be prepended to the output of a later successful one. -RUN retry apt-get -o Acquire::Retries=3 update \ +# APT normally treats transient index-download failures as warnings and exits +# zero. Make every update fail on those errors so the outer retry can run. +RUN retry apt-get -o Acquire::Retries=3 --error-on=any update \ && retry apt-get -o Acquire::Retries=3 install -y --no-install-recommends wget gnupg ca-certificates \ && retry wget -q -O /etc/apt/trusted.gpg.d/apt.llvm.org.asc https://apt.llvm.org/llvm-snapshot.gpg.key \ && echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" > /etc/apt/sources.list.d/llvm-21.list \ - && retry apt-get -o Acquire::Retries=3 update \ + && retry apt-get -o Acquire::Retries=3 --error-on=any update \ && retry apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ clang-21 libclang-rt-21-dev llvm-21 \ && ln -sf /usr/bin/clang-21 /usr/bin/clang \ && ln -sf /usr/bin/clang++-21 /usr/bin/clang++ -RUN retry apt-get -o Acquire::Retries=3 update \ +RUN retry apt-get -o Acquire::Retries=3 --error-on=any update \ && retry apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ cmake \ ninja-build \ From 25383bc7a462f50255b31fc9e75c772ba0025547 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 23:39:20 +0200 Subject: [PATCH 12/34] fix: avoid null arguments in empty sort and edge-copy operations An empty project cache leaves the list_projects record array unallocated. Skip qsort when there is at most one record. Workers that emit no deferred semantic edges likewise have no allocated source buffer; skip their zero-length memcpy during the merge. Add an empty-cache response regression covering JSON and tree output. The Linux sanitizer proof rejects the original qsort call, and the existing pipeline suite with 64 workers reports the original empty-worker copy. Both diagnostics disappear with the guards restored. Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 6 +++- src/pipeline/pass_semantic_edges.c | 10 +++++-- tests/test_mcp.c | 48 ++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c877d5110..64789d91b 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3044,7 +3044,11 @@ static char *handle_list_projects(cbm_mcp_server_t *srv, const char *args) { free(records); return cbm_mcp_text_result("out of memory while listing projects", true); } - qsort(records, (size_t)record_count, sizeof(*records), project_record_compare); + /* An empty cache never allocates the record array, and qsort's base is + * declared nonnull (glibc): skip the sort when there is nothing to order. */ + if (record_count > 1) { + qsort(records, (size_t)record_count, sizeof(*records), project_record_compare); + } int limit = cbm_mcp_get_int_arg(args, "limit", 50); int offset = cbm_mcp_get_int_arg(args, "offset", 0); diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 05a7387a9..14597289e 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -1156,9 +1156,13 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf } int n = 0; for (int w = 0; w < worker_count; w++) { - memcpy(&pairs[n], worker_bufs[w].edges, - (size_t)worker_bufs[w].count * sizeof(deferred_edge_t)); - n += worker_bufs[w].count; + /* A worker that emitted no pair never allocated its buffer, and + * memcpy's source is declared nonnull (glibc) even for zero bytes. */ + if (worker_bufs[w].count > 0) { + memcpy(&pairs[n], worker_bufs[w].edges, + (size_t)worker_bufs[w].count * sizeof(deferred_edge_t)); + n += worker_bufs[w].count; + } deferred_buf_free(&worker_bufs[w]); } qsort(pairs, (size_t)n, sizeof(deferred_edge_t), cmp_deferred_edge_canonical); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 22af972af..b78b0fe51 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -2349,6 +2349,53 @@ TEST(tool_list_projects_empty) { PASS(); } +/* A cache directory that exists but holds no project is the first-run state. + * list_projects must answer it with a well-formed empty page in both + * encodings: zero rows, zero totals, no further page, and the indexing hint. + * + * This is also the one input on which the record array is never allocated. + * The assertions are the behavioural half and hold on every platform; that the + * sort is not handed a null base is visible only where the C library declares + * it nonnull and the sanitizer halts on the report (see the same note on + * pass_similarity_empty_graph_no_entries). Do not read a green run on a + * recovering lane as proof that the guard is still in place. */ +TEST(tool_list_projects_empty_cache_returns_wellformed_empty_page) { + mcp_search_cache_t cache; + ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-list-empty")); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + + char *json_response = + srv ? cbm_mcp_handle_tool(srv, "list_projects", "{\"format\":\"json\"}") : NULL; + char *json_text = json_response ? extract_text_content(json_response) : NULL; + bool json_empty_page = + json_text && strstr(json_text, "\"projects\":[]") && strstr(json_text, "\"total\":0") && + strstr(json_text, "\"returned\":0") && strstr(json_text, "\"has_more\":false") && + !strstr(json_text, "next_offset") && strstr(json_text, "No projects indexed"); + bool json_not_error = json_response && !strstr(json_response, "\"isError\":true"); + + char *tree_response = srv ? cbm_mcp_handle_tool(srv, "list_projects", "{}") : NULL; + char *tree_text = tree_response ? extract_text_content(tree_response) : NULL; + bool tree_empty_page = + tree_text && strstr(tree_text, "projects: 0 ") && strstr(tree_text, "total: 0\n") && + strstr(tree_text, "returned: 0\n") && strstr(tree_text, "has_more: false\n") && + !strstr(tree_text, "next_offset") && strstr(tree_text, "No projects indexed"); + bool tree_not_error = tree_response && !strstr(tree_response, "\"isError\":true"); + + free(json_text); + free(json_response); + free(tree_text); + free(tree_response); + cbm_mcp_server_free(srv); + ASSERT_TRUE(mcp_search_cache_close(&cache)); + + ASSERT_NOT_NULL(srv); + ASSERT_TRUE(json_not_error); + ASSERT_TRUE(json_empty_page); + ASSERT_TRUE(tree_not_error); + ASSERT_TRUE(tree_empty_page); + PASS(); +} + TEST(tool_get_graph_schema_empty) { cbm_mcp_server_t *srv = setup_mcp_with_data(); @@ -21095,6 +21142,7 @@ SUITE(mcp) { /* Tool handlers */ RUN_TEST(tool_list_projects_empty); + RUN_TEST(tool_list_projects_empty_cache_returns_wellformed_empty_page); RUN_TEST(tool_get_graph_schema_empty); RUN_TEST(tool_unknown_tool); RUN_TEST(tool_compare_graphs_registered_issue525); From 78772eb3011f6d5c39bd60f02f3d6cdef5b6bae0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 18:37:19 +0200 Subject: [PATCH 13/34] test: checkpoint cross-platform harness fixes Isolate the watcher-disabled runtime, use a deterministic Python responder for the fuzz harness environment probe, and align the version metadata contract with the current Scoop manifest. Signed-off-by: Martin Vogel --- tests/test_runtime_isolation_contract.sh | 1 + tests/test_security_fuzz_harness.sh | 39 +++++++++++++++--------- tests/test_version_metadata_contract.sh | 3 +- tests/test_watcher_disabled.sh | 19 +++++++++--- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/tests/test_runtime_isolation_contract.sh b/tests/test_runtime_isolation_contract.sh index 217230c7f..0d6865a49 100644 --- a/tests/test_runtime_isolation_contract.sh +++ b/tests/test_runtime_isolation_contract.sh @@ -93,6 +93,7 @@ ENTRY_POINTS=( scripts/security-install.sh scripts/security-fuzz.sh scripts/security-fuzz-random.sh scripts/security-network.sh tests/test_parent_watchdog.sh tests/test_worker_watchdog.sh tests/test_worker_error_response.sh tests/test_hook_conflict_notice.sh + tests/test_watcher_disabled.sh ) for entry in "${ENTRY_POINTS[@]}"; do grep -q 'test-runtime.sh' "$ROOT/$entry" || fail "$entry does not source the helper" diff --git a/tests/test_security_fuzz_harness.sh b/tests/test_security_fuzz_harness.sh index 713107b1f..79f43ad3e 100644 --- a/tests/test_security_fuzz_harness.sh +++ b/tests/test_security_fuzz_harness.sh @@ -41,24 +41,35 @@ if "$ROOT/scripts/security-fuzz.sh" "$ECHO_ONLY" \ exit 1 fi +# One python3 process answers the requests; the harness needs python3 for its +# interactive case anyway. A shell `read` of the 1 MB request costs seconds +# where the shell runs emulated, and the harness's 10-second hang limit then +# decides this test instead of its assertions. +ENV_RESPONDER="$WORKDIR/environment-probe-responder.py" +cat > "$ENV_RESPONDER" <<'EOF' +import re +import sys + +# Echo a JSON-RPC result for every request with a numeric id. This keeps the +# fixture compatible with both the current fixed ids and a future per-case +# acknowledgement id without depending on the malformed payload itself: the +# last id of a line is the one that counts. +REQUEST_ID = re.compile(rb'"id"\s*:\s*([0-9]+)') + +for line in sys.stdin.buffer: + ids = REQUEST_ID.findall(line) + if not ids: + continue + result = b'{"isError":true}' if b'"name":"index_repository"' in line else b'{}' + sys.stdout.buffer.write(b'{"jsonrpc":"2.0","id":' + ids[-1] + b',"result":' + result + b'}\n') + sys.stdout.buffer.flush() +EOF + ENV_PROBE="$WORKDIR/environment-probe-mcp" cat > "$ENV_PROBE" <<'EOF' #!/usr/bin/env bash printf '%s\t%s\t%s\n' "${HOME-}" "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_FUZZ_ENV_PROBE" - -# Echo a JSON-RPC result for every request with a numeric id. This keeps the -# fixture compatible with both the current fixed ids and a future per-case -# acknowledgement id without depending on the malformed payload itself. -while IFS= read -r line; do - id=$(printf '%s\n' "$line" | sed -n 's/.*"id"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p') - if [[ -n "$id" ]]; then - if [[ "$line" == *'"name":"index_repository"'* ]]; then - printf '{"jsonrpc":"2.0","id":%s,"result":{"isError":true}}\n' "$id" - else - printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" - fi - fi -done +exec python3 "${BASH_SOURCE[0]%/*}/environment-probe-responder.py" EOF chmod +x "$ENV_PROBE" diff --git a/tests/test_version_metadata_contract.sh b/tests/test_version_metadata_contract.sh index fa66caeaa..05655a148 100644 --- a/tests/test_version_metadata_contract.sh +++ b/tests/test_version_metadata_contract.sh @@ -107,8 +107,8 @@ SURFACES=( "pkg/go/cmd/codebase-memory-mcp/main.go|release" "pkg/chocolatey/codebase-memory-mcp.nuspec|release" "pkg/chocolatey/tools/chocolateyInstall.ps1|release" + "pkg/scoop/codebase-memory-mcp.json|release" "pkg/homebrew/Formula/codebase-memory-mcp.rb|pin:0.10.3" - "pkg/scoop/codebase-memory-mcp.json|pin:0.11.0" "pkg/aur/PKGBUILD|pin:0.8.1" "pkg/aur/.SRCINFO|pin:0.8.1" ) @@ -119,7 +119,6 @@ SURFACES=( # AND every sha256 it pins, then move the entry to "release" above. PIN_REASONS=( "pkg/homebrew/Formula/codebase-memory-mcp.rb|pins 4 per-asset sha256 (darwin/linux x arm/intel); last re-pinned for v0.10.3" - "pkg/scoop/codebase-memory-mcp.json|pins 2 per-asset sha256 (Windows amd64 + arm64 zip); last re-pinned for v0.11.0" "pkg/aur/PKGBUILD|pins sha256sums_x86_64 + sha256sums_aarch64; last re-pinned for v0.8.1" "pkg/aur/.SRCINFO|generated from PKGBUILD, so it must move with it, not before it" ) diff --git a/tests/test_watcher_disabled.sh b/tests/test_watcher_disabled.sh index 23d91ae74..bc6209350 100755 --- a/tests/test_watcher_disabled.sh +++ b/tests/test_watcher_disabled.sh @@ -39,18 +39,27 @@ esac [[ -x "${BINARY}" ]] || { echo "missing binary: ${BINARY}" >&2; exit 2; } command -v git >/dev/null 2>&1 || { echo "git required for fixture" >&2; exit 2; } -work="$(mktemp -d)" +# A private runtime directory, not only a private cache: daemon admission is +# decided per runtime directory, so in the account-wide one a released CBM that +# a developer has in use refuses this build ("a conflicting CBM process is +# active") and the test could never start. With the helper's root these daemons +# never meet a developer's real one. Its root is short enough for the socket +# path; everything this test creates lives under it. +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +work="${CBM_TEST_RUNTIME_ROOT}/work" +mkdir "${work}" -# Every run gets its own CBM_CACHE_DIR, and the daemon endpoint is derived from -# it, so these daemons are private to the test and can never touch a developer's -# real one. Retire them on any exit path regardless. +# Every run gets its own CBM_CACHE_DIR (and so its own daemon log). Retire the +# daemons on any exit path, then let the helper remove the root. cleanup() { local cache for cache in "${work}"/cache-*; do [[ -d "${cache}" ]] || continue CBM_CACHE_DIR="${cache}" "${BINARY}" daemon stop >/dev/null 2>&1 || true done - rm -rf "${work}" + cbm_test_runtime_cleanup "${BINARY}" } trap cleanup EXIT From 6b1cc878abbf532b6761feda444aa9f630386dd2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 18:38:13 +0200 Subject: [PATCH 14/34] fix(pipeline): resolve overrides through nearest declaring ancestors Preserve all equally near targets and guard Python inheritance against external import collisions. Refs #1278. Signed-off-by: Martin Vogel --- src/pipeline/pass_lsp_cross.c | 80 ++++---- src/pipeline/pass_parallel.c | 25 ++- src/pipeline/pass_pkgmap.c | 26 +++ src/pipeline/pass_semantic.c | 253 +++++++++++++++++++++---- src/pipeline/pipeline_internal.h | 11 ++ tests/test_edge_imports.c | 169 +++++++++++++++++ tests/test_pipeline.c | 314 +++++++++++++++++++++++++++++++ tests/test_registry.c | 37 ++++ 8 files changed, 844 insertions(+), 71 deletions(-) diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 66751a682..5fb4bfd95 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -220,25 +220,47 @@ static bool pxc_base_strategy_is_weak(const char *strategy) { strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; } +/* One file's base-resolution scope: the registry + import map pass_semantic + * uses, plus the facts the external-base gate reads. NULL (the surface-probe + * path) keeps every base at its raw spelling. */ +typedef struct { + const cbm_registry_t *reg; + const char **imp_keys; + const char **imp_vals; + int imp_count; + const CBMImportArray *imports; + const cbm_gbuf_t *gbuf; + const char *project_name; + const char *rel; + CBMLanguage lang; +} pxc_base_scope_t; + /* Resolve one base-class spelling to a project QN. Mirrors * pass_semantic.c::resolve_as_class — same registry, same type-like veto — * then additionally rejects weak short-name strategies (see above). * Returns NULL when the base is not a confidently-known project type; * stdlib and third-party bases land here and keep their raw spelling. */ -static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *raw, - const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { - if (!reg || !raw || !raw[0]) { +static const char *pxc_resolve_base_qn(const pxc_base_scope_t *bs, const char *raw, + const char *module_qn) { + if (!bs || !bs->reg || !raw || !raw[0]) { return NULL; } - cbm_resolution_t res = cbm_registry_resolve(reg, raw, module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t res = + cbm_registry_resolve(bs->reg, raw, module_qn, bs->imp_keys, bs->imp_vals, bs->imp_count); if (!res.qualified_name || !res.qualified_name[0]) { return NULL; } if (pxc_base_strategy_is_weak(res.strategy)) { return NULL; } - if (!cbm_label_is_type_like(cbm_registry_label_of(reg, res.qualified_name))) { + if (!cbm_label_is_type_like(cbm_registry_label_of(bs->reg, res.qualified_name))) { + return NULL; + } + /* `unittest.TestCase` under an external `import unittest` survives the + * weak-strategy veto through the same-module suffix fallback; the shared + * external-base gate keeps it at its source spelling. */ + if (cbm_python_external_base_contradicts(bs->lang, bs->imports, raw, res.qualified_name, + bs->gbuf, bs->project_name, bs->rel)) { return NULL; } return res.qualified_name; @@ -248,8 +270,7 @@ static const char *pxc_resolve_base_qn(const cbm_registry_t *reg, const char *ra * source spelling. Unresolved entries pass through verbatim so a base the * registry does not know keeps working exactly as before. */ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, - const cbm_registry_t *reg, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const pxc_base_scope_t *bs, const char *module_qn) { if (!bases || !bases[0]) { return NULL; } @@ -263,8 +284,7 @@ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, return pxc_join_pipe(arena, bases); } for (int i = 0; i < count; i++) { - const char *qn = - pxc_resolve_base_qn(reg, bases[i], module_qn, imp_keys, imp_vals, imp_count); + const char *qn = pxc_resolve_base_qn(bs, bases[i], module_qn); resolved[i] = qn ? qn : bases[i]; } resolved[count] = NULL; @@ -390,8 +410,7 @@ static const char *pxc_qn_leaf(const char *name) { * pointers into src and into `arena` for synthesised composites. */ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const char *module_qn, const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst, - const cbm_registry_t *reg, const char **imp_keys, - const char **imp_vals, int imp_count) { + const pxc_base_scope_t *bs) { const char *label = pxc_map_label(arena, src->label); if (!label || !src->qualified_name || !src->name) return -1; @@ -417,9 +436,8 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch /* Languages whose cross registrars read embedded_types as QNs get their * bases resolved against the project registry; everyone else keeps the * raw source spelling their own registrar already knows how to handle. */ - dst->embedded_types = (reg && pxc_lang_resolves_base_qns(lang)) - ? pxc_join_base_qns(arena, src->base_classes, reg, module_qn, - imp_keys, imp_vals, imp_count) + dst->embedded_types = (bs && bs->reg && pxc_lang_resolves_base_qns(lang)) + ? pxc_join_base_qns(arena, src->base_classes, bs, module_qn) : pxc_join_pipe(arena, src->base_classes); dst->signature_param_types = NULL; dst->signature_param_count = 0; @@ -593,11 +611,10 @@ typedef struct { } pxc_py_field_t; static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs, - int start, int end, const cbm_registry_t *reg, - const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count) { + int start, int end, const pxc_base_scope_t *bs, + const char *module_qn) { int nf = result ? result->field_types.count : 0; - if (!arena || nf == 0 || !defs || start >= end || !reg) { + if (!arena || nf == 0 || !defs || start >= end || !bs || !bs->reg) { return; } CBMHashTable *by_qn = cbm_ht_create((uint32_t)(end - start)); @@ -622,8 +639,7 @@ static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result fields[f].type_qn = NULL; CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; - const char *qn = - name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + const char *qn = name ? pxc_resolve_base_qn(bs, name, module_qn) : NULL; if (!qn || !ft->field_name || !ft->field_name[0]) { continue; } @@ -744,27 +760,27 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are * that consume resolved base QNs, and only when a caller supplied the * pipeline context (the surface-probe path passes NULL and keeps the * raw spelling). */ - const cbm_registry_t *base_reg = NULL; - const char **imp_keys = NULL; - const char **imp_vals = NULL; - int imp_count = 0; + pxc_base_scope_t bs = {.imports = &fr->imports, + .project_name = project_name, + .rel = files[fi].rel_path, + .lang = files[fi].language}; if (ctx && ctx->registry && pxc_lang_resolves_base_qns(files[fi].language)) { - base_reg = ctx->registry; + bs.reg = ctx->registry; + bs.gbuf = ctx->gbuf; cbm_pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path, - files[fi].language, fr, &imp_keys, &imp_vals, &imp_count); + files[fi].language, fr, &bs.imp_keys, &bs.imp_vals, + &bs.imp_count); } for (int di = 0; di < fr->defs.count; di++) { if (pxc_build_lsp_def(arena, &fr->defs.items[di], def_modules[fi], namespace_name, - files[fi].language, &defs[idx], base_reg, imp_keys, imp_vals, - imp_count) == 0) { + files[fi].language, &defs[idx], bs.reg ? &bs : NULL) == 0) { idx++; } } if (files[fi].language == CBM_LANG_PYTHON) { - pxc_fold_py_field_types(arena, fr, defs, file_start, idx, base_reg, def_modules[fi], - imp_keys, imp_vals, imp_count); + pxc_fold_py_field_types(arena, fr, defs, file_start, idx, &bs, def_modules[fi]); } - cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ + cbm_pxc_free_import_map(bs.imp_keys, bs.imp_vals, bs.imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a91acaf83..c5cedf445 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3357,16 +3357,27 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi } } +/* The per-file facts the base-class gate needs beyond the import map. */ +typedef struct { + const CBMImportArray *imports; + const char *rel; + CBMLanguage lang; +} pp_file_scope_t; + /* Resolve base_classes → INHERITS edges for one definition. */ static void resolve_def_inherits(resolve_ctx_t *rc, resolve_worker_state_t *ws, const CBMDefinition *def, const cbm_gbuf_node_t *node, - const char *mq, const char **ik, const char **iv, int ic) { + const pp_file_scope_t *fs, const char *mq, const char **ik, + const char **iv, int ic) { if (!def->base_classes) { return; } for (int b = 0; def->base_classes[b]; b++) { const char *bqn = resolve_as_class(rc->registry, def->base_classes[b], mq, ik, iv, ic); - if (!bqn) { + /* Same external-base gate as the sequential semantic pass. */ + if (!bqn || + cbm_python_external_base_contradicts(fs->lang, fs->imports, def->base_classes[b], bqn, + rc->main_gbuf, rc->project_name, fs->rel)) { continue; } const cbm_gbuf_node_t *bn = cbm_gbuf_find_by_qn(rc->main_gbuf, bqn); @@ -3453,8 +3464,10 @@ static void resolve_def_decorators(resolve_ctx_t *rc, resolve_worker_state_t *ws /* Resolve INHERITS + DECORATES + IMPLEMENTS for one file. */ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, - CBMFileResult *result, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + CBMFileResult *result, const char *rel, CBMLanguage lang, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + const pp_file_scope_t fs = {&result->imports, rel, lang}; for (int d = 0; d < result->defs.count; d++) { CBMDefinition *def = &result->defs.items[d]; if (!def->qualified_name) { @@ -3464,7 +3477,7 @@ static void resolve_file_semantic(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (!node) { continue; } - resolve_def_inherits(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); + resolve_def_inherits(rc, ws, def, node, &fs, module_qn, imp_keys, imp_vals, imp_count); resolve_def_decorators(rc, ws, def, node, module_qn, imp_keys, imp_vals, imp_count); } for (int t = 0; t < result->impl_traits.count; t++) { @@ -3813,7 +3826,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* ── INHERITS + DECORATES + IMPLEMENTS ──────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_semantic(rc, ws, result, module_qn, imp_keys, imp_vals, imp_count); + resolve_file_semantic(rc, ws, result, rel, lang, module_qn, imp_keys, imp_vals, imp_count); atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, memory_order_relaxed); diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index dc5e46db0..152f111d0 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -1676,6 +1676,11 @@ bool cbm_python_import_binding_contradicts(const CBMImportArray *imports, const /* `import a.b` binds the ROOT package `a`: the callee already * spells its own full dotted path (`a.b.f()`). */ snprintf(full, sizeof(full), "%s", callee_name); + } else if (!as && !path_leaf_seg && full[0] != '.' && strcmp(full, imp->local_name) != 0) { + /* `import m as y` (the extractor stores module_path `m`, local + * `y`; a from-import always spells a dotted path): `y` IS module + * `m`, so `y.T` spells `m.T`. */ + snprintf(full, sizeof(full), "%s%s", imp->module_path, callee_name + root_len); } /* Otherwise (`from m import x [as y]`) only the import's own module * chain is evidence: members reached THROUGH x (`x.objects.create`) @@ -1691,6 +1696,27 @@ bool cbm_python_import_binding_contradicts(const CBMImportArray *imports, const return bound; } +/* Base-class twin of the #2127 call gate. `class SimpleTestCase( + * unittest.TestCase)` under `import unittest` names the STDLIB base; the + * registry's same-module suffix fallback (`unittest.TestCase` -> module's + * `TestCase`) or a short-name strategy bound it to a same-named PROJECT class + * (django: an inheritance cycle SimpleTestCase -> TestCase -> + * TransactionTestCase -> SimpleTestCase, plus fabricated OVERRIDEs). An + * external import binding (never materialized as an IMPORTS edge) whose + * module chain the target does not spell can never be that target, whatever + * the registry strategy. Exact metadata import maps remain valid when the + * target spells the imported module chain. Every base resolver (sequential + * + parallel semantic, cross-LSP pxc) calls this ONE + * gate so the venues cannot diverge. Python only. */ +bool cbm_python_external_base_contradicts(CBMLanguage lang, const CBMImportArray *imports, + const char *base_spelling, const char *base_qn, + const cbm_gbuf_t *gbuf, const char *project_name, + const char *rel_path) { + return lang == CBM_LANG_PYTHON && + cbm_python_import_binding_contradicts(imports, base_spelling, base_qn, gbuf, + project_name, rel_path); +} + /* #2127: whether a Python import's module lives in this project. Relative * imports always do; otherwise the module part (the path minus the imported * name) must resolve to a graph node. Strategy 1 has already failed on the diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab0083..128576cf3 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -611,6 +611,7 @@ static void resolve_decorator(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *no } static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, + const cbm_file_info_t *fi, const CBMFileResult *result, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, int *inherits_count, int *decorates_count) { @@ -625,8 +626,10 @@ static void sem_process_def_edges(cbm_pipeline_ctx_t *ctx, const CBMDefinition * for (int b = 0; def->base_classes[b]; b++) { const char *base_qn = resolve_as_class(ctx->registry, def->base_classes[b], module_qn, imp_keys, imp_vals, imp_count); - if (!base_qn) { - continue; + if (!base_qn || cbm_python_external_base_contradicts( + fi->language, &result->imports, def->base_classes[b], base_qn, + ctx->gbuf, ctx->project_name, fi->rel_path)) { + continue; /* unresolved, or an external base (`unittest.TestCase`) */ } const cbm_gbuf_node_t *base_node = cbm_gbuf_find_by_qn(ctx->gbuf, base_qn); if (base_node && node->id != base_node->id) { @@ -736,8 +739,8 @@ int cbm_pipeline_pass_semantic(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *f /* ── INHERITS + DECORATES from definitions ──────────────── */ for (int d = 0; d < result->defs.count; d++) { - sem_process_def_edges(ctx, &result->defs.items[d], module_qn, imp_keys, imp_vals, - imp_count, &inherits_count, &decorates_count); + sem_process_def_edges(ctx, &result->defs.items[d], &files[i], result, module_qn, + imp_keys, imp_vals, imp_count, &inherits_count, &decorates_count); } /* ── IMPLEMENTS from impl_traits (Rust) ─────────────────── */ @@ -770,31 +773,111 @@ const char *cbm_semantic_base_edge_type(const cbm_gbuf_node_t *base_node) { : "INHERITS"; } -/* Create OVERRIDE edges from one class's methods to same-named methods of one - * explicit base (interface or superclass). */ -static int override_match_methods(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t *cls, - const cbm_gbuf_node_t *base) { - const cbm_gbuf_edge_t **cls_dm = NULL; - int cls_dm_count = 0; - cbm_gbuf_find_edges_by_source_type(ctx->gbuf, cls->id, "DEFINES_METHOD", &cls_dm, - &cls_dm_count); - if (cls_dm_count == 0) { - return 0; +/* ── Explicit OVERRIDE: nearest declaring ancestor (#1278) ────────── + * + * A method overrides the same-named method of its NEAREST ancestor that + * declares one, not only of its direct base: `Leaf(Intermediate(Processor))` + * with an Intermediate that redeclares nothing still binds + * Leaf.process -> Processor.process. + * + * The walk is level-synchronous over explicit IMPLEMENTS/INHERITS edges: + * level 1 is every direct base, level k+1 every not-yet-visited base of + * level k. A method is bound to every declaring ancestor on the FIRST level + * that has one and is not looked up further, so the edge set does not depend + * on edge or node order (a diamond's two equally near declarations both + * bind; its shared root binds once, via insert-edge dedup). Direct bases are + * level 1, so every edge the direct-base-only matcher made is still made. + * A visited set makes every ancestor enter the walk at most once: cycles and + * self-loops in a malformed hierarchy terminate, and the level count can + * never exceed the node count (the check below is only that invariant). */ + +typedef struct { + int64_t *ids; + int count; + int cap; +} ovr_id_list_t; + +static bool ovr_ids_push(ovr_id_list_t *list, int64_t id) { + if (list->count == list->cap) { + int cap = list->cap ? list->cap * 2 : 16; + int64_t *grown = + cbm_realloc(CBM_MEM_CLASS_SEMANTIC, list->ids, (size_t)cap * sizeof(int64_t)); + if (!grown) { + return false; + } + list->ids = grown; + list->cap = cap; } + list->ids[list->count++] = id; + return true; +} + +static int ovr_cmp_id(const void *a, const void *b) { + int64_t x = *(const int64_t *)a; + int64_t y = *(const int64_t *)b; + return (x > y) - (x < y); +} + +/* Sort + unique in place. */ +static void ovr_ids_normalize(ovr_id_list_t *list) { + if (list->count < 2) { + return; + } + qsort(list->ids, (size_t)list->count, sizeof(int64_t), ovr_cmp_id); + int w = 1; + for (int r = 1; r < list->count; r++) { + if (list->ids[r] != list->ids[w - 1]) { + list->ids[w++] = list->ids[r]; + } + } + list->count = w; +} + +static bool ovr_ids_has(const ovr_id_list_t *sorted, int64_t id) { + return sorted->count > 0 && + bsearch(&id, sorted->ids, (size_t)sorted->count, sizeof(int64_t), ovr_cmp_id) != NULL; +} + +static const char *const ovr_base_edge_types[] = {"IMPLEMENTS", "INHERITS"}; +enum { OVR_BASE_EDGE_TYPE_COUNT = 2 }; + +/* Append every explicit base of `node` to `out`. */ +static bool ovr_push_bases(const cbm_gbuf_t *gb, int64_t node, ovr_id_list_t *out) { + for (int t = 0; t < OVR_BASE_EDGE_TYPE_COUNT; t++) { + const cbm_gbuf_edge_t **edges = NULL; + int n = 0; + cbm_gbuf_find_edges_by_source_type(gb, node, ovr_base_edge_types[t], &edges, &n); + for (int e = 0; e < n; e++) { + if (!ovr_ids_push(out, edges[e]->target_id)) { + return false; + } + } + } + return true; +} + +/* Bind each still-unbound method of the class to the same-named methods of + * one ancestor. Marks nothing: a level's bindings are committed by the caller + * after the whole level ran, so equally near declarations all bind. */ +static int ovr_match_ancestor(cbm_gbuf_t *gb, const cbm_gbuf_edge_t **cls_dm, int cls_dm_count, + const bool *bound, bool *bound_now, int64_t ancestor) { const cbm_gbuf_edge_t **base_dm = NULL; int base_dm_count = 0; - cbm_gbuf_find_edges_by_source_type(ctx->gbuf, base->id, "DEFINES_METHOD", &base_dm, - &base_dm_count); + cbm_gbuf_find_edges_by_source_type(gb, ancestor, "DEFINES_METHOD", &base_dm, &base_dm_count); int created = 0; for (int c = 0; c < cls_dm_count; c++) { - const cbm_gbuf_node_t *cm = cbm_gbuf_find_by_id(ctx->gbuf, cls_dm[c]->target_id); + if (bound[c]) { + continue; + } + const cbm_gbuf_node_t *cm = cbm_gbuf_find_by_id(gb, cls_dm[c]->target_id); if (!cm || !cm->name) { continue; } for (int b = 0; b < base_dm_count; b++) { - const cbm_gbuf_node_t *bm = cbm_gbuf_find_by_id(ctx->gbuf, base_dm[b]->target_id); + const cbm_gbuf_node_t *bm = cbm_gbuf_find_by_id(gb, base_dm[b]->target_id); if (bm && bm->name && cm->id != bm->id && strcmp(cm->name, bm->name) == 0) { - cbm_gbuf_insert_edge(ctx->gbuf, cm->id, bm->id, "OVERRIDE", "{}"); + cbm_gbuf_insert_edge(gb, cm->id, bm->id, "OVERRIDE", "{}"); + bound_now[c] = true; created++; break; } @@ -803,29 +886,133 @@ static int override_match_methods(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t return created; } +typedef struct { + ovr_id_list_t visited; /* sorted */ + ovr_id_list_t level; + ovr_id_list_t next; + bool *bound; + bool *bound_now; + int flags_cap; +} ovr_walk_t; + +static void ovr_walk_free(ovr_walk_t *w) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, w->visited.ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, w->level.ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, w->next.ids); + cbm_free(CBM_MEM_CLASS_SEMANTIC, w->bound); + cbm_free(CBM_MEM_CLASS_SEMANTIC, w->bound_now); +} + +static bool ovr_walk_reserve_flags(ovr_walk_t *w, int n) { + if (n <= w->flags_cap) { + return true; + } + bool *b = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, w->bound, (size_t)n * sizeof(bool)); + if (!b) { + return false; + } + w->bound = b; + bool *bn = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, w->bound_now, (size_t)n * sizeof(bool)); + if (!bn) { + return false; + } + w->bound_now = bn; + w->flags_cap = n; + return true; +} + +/* Drop from `next` every id already visited, then add the rest to visited. */ +static bool ovr_walk_advance(ovr_walk_t *w) { + ovr_ids_normalize(&w->next); + w->level.count = 0; + for (int i = 0; i < w->next.count; i++) { + int64_t id = w->next.ids[i]; + if (!ovr_ids_has(&w->visited, id) && !ovr_ids_push(&w->level, id)) { + return false; + } + } + for (int i = 0; i < w->level.count; i++) { + if (!ovr_ids_push(&w->visited, w->level.ids[i])) { + return false; + } + } + ovr_ids_normalize(&w->visited); + w->next.count = 0; + return true; +} + +/* All OVERRIDE edges of one class: walk its ancestors level by level until + * every method is bound or the ancestors run out. */ +static int ovr_class_overrides(cbm_gbuf_t *gb, int64_t cls_id, ovr_walk_t *w) { + const cbm_gbuf_edge_t **cls_dm = NULL; + int cls_dm_count = 0; + cbm_gbuf_find_edges_by_source_type(gb, cls_id, "DEFINES_METHOD", &cls_dm, &cls_dm_count); + if (cls_dm_count == 0 || !ovr_walk_reserve_flags(w, cls_dm_count)) { + return 0; + } + memset(w->bound, 0, (size_t)cls_dm_count * sizeof(bool)); + memset(w->bound_now, 0, (size_t)cls_dm_count * sizeof(bool)); + w->visited.count = 0; + w->next.count = 0; + if (!ovr_ids_push(&w->visited, cls_id) || !ovr_push_bases(gb, cls_id, &w->next) || + !ovr_walk_advance(w)) { + return 0; + } + int created = 0; + int unbound = cls_dm_count; + const int max_levels = cbm_gbuf_node_count(gb); /* invariant, never reached */ + for (int depth = 0; w->level.count > 0 && unbound > 0 && depth < max_levels; depth++) { + for (int i = 0; i < w->level.count; i++) { + created += ovr_match_ancestor(gb, cls_dm, cls_dm_count, w->bound, w->bound_now, + w->level.ids[i]); + if (!ovr_push_bases(gb, w->level.ids[i], &w->next)) { + return created; + } + } + for (int c = 0; c < cls_dm_count; c++) { + if (w->bound_now[c] && !w->bound[c]) { + w->bound[c] = true; + unbound--; + } + } + if (!ovr_walk_advance(w)) { + return created; + } + } + return created; +} + int cbm_pipeline_override_explicit(cbm_pipeline_ctx_t *ctx) { if (!ctx || !ctx->gbuf) { return 0; } - int created = 0; - static const char *base_edge_types[] = {"IMPLEMENTS", "INHERITS"}; - for (size_t t = 0; t < sizeof(base_edge_types) / sizeof(base_edge_types[0]); t++) { + /* Every class with at least one explicit base, once, in id order. */ + ovr_id_list_t classes = {0}; + for (int t = 0; t < OVR_BASE_EDGE_TYPE_COUNT; t++) { const cbm_gbuf_edge_t **edges = NULL; int edge_count = 0; - cbm_gbuf_find_edges_by_type(ctx->gbuf, base_edge_types[t], &edges, &edge_count); + cbm_gbuf_find_edges_by_type(ctx->gbuf, ovr_base_edge_types[t], &edges, &edge_count); for (int e = 0; e < edge_count; e++) { - const cbm_gbuf_node_t *cls = cbm_gbuf_find_by_id(ctx->gbuf, edges[e]->source_id); - const cbm_gbuf_node_t *base = cbm_gbuf_find_by_id(ctx->gbuf, edges[e]->target_id); - if (!cls || !base) { - continue; + if (!ovr_ids_push(&classes, edges[e]->source_id)) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, classes.ids); + return 0; } - /* Go's implicit satisfaction already emits OVERRIDE with interface - * semantics; running both would double-cover .go sources. */ - if (cls->file_path && fp_ends_with(cls->file_path, ".go")) { - continue; - } - created += override_match_methods(ctx, cls, base); } } + ovr_ids_normalize(&classes); + + int created = 0; + ovr_walk_t walk = {0}; + for (int i = 0; i < classes.count; i++) { + const cbm_gbuf_node_t *cls = cbm_gbuf_find_by_id(ctx->gbuf, classes.ids[i]); + /* Go's implicit satisfaction already emits OVERRIDE with interface + * semantics; running both would double-cover .go sources. */ + if (!cls || (cls->file_path && fp_ends_with(cls->file_path, ".go"))) { + continue; + } + created += ovr_class_overrides(ctx->gbuf, cls->id, &walk); + } + ovr_walk_free(&walk); + cbm_free(CBM_MEM_CLASS_SEMANTIC, classes.ids); return created; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae9..03182d833 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -317,6 +317,17 @@ bool cbm_python_import_binding_contradicts(const CBMImportArray *imports, const const char *resolved_qn, const cbm_gbuf_t *gbuf, const char *project_name, const char *rel_path); +/* Base-class twin: true when a Python base spelling (`unittest.TestCase`, + * `TestCase` under `from unittest import TestCase`, `ut.TestCase` under + * `import unittest as ut`) is bound by an EXTERNAL import whose chain + * contradicts `base_qn` -- the INHERITS / embedded-type resolution must then + * leave the base unresolved. False for every other language. Shared by + * pass_semantic.c, pass_parallel.c and pass_lsp_cross.c. */ +bool cbm_python_external_base_contradicts(CBMLanguage lang, const CBMImportArray *imports, + const char *base_spelling, const char *base_qn, + const cbm_gbuf_t *gbuf, const char *project_name, + const char *rel_path); + /* Check if a file path is worth tracking for git history analysis. */ bool cbm_is_trackable_file(const char *path); diff --git a/tests/test_edge_imports.c b/tests/test_edge_imports.c index 097cebafe..99a3e0035 100644 --- a/tests/test_edge_imports.c +++ b/tests/test_edge_imports.c @@ -752,6 +752,174 @@ TEST(ei_py_external_import_never_binds_project_symbol) { PASS(); } +/* Base-class helper: INHERITS edges out of the (single) Class named `src` + * (return value), and how many land on a node whose QN ends with + * `want_qn_suffix` (*to_want); -1 when the class is missing. */ +static int ei_inherits_out(cbm_store_t *store, const char *project, const char *src, + const char *want_qn_suffix, int *to_want) { + *to_want = 0; + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_name(store, project, src, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + int64_t id = 0; + for (int i = 0; i < count; i++) { + if (nodes[i].label && strcmp(nodes[i].label, "Class") == 0) { + id = nodes[i].id; + } + } + cbm_store_free_nodes(nodes, count); + if (id == 0) { + return -1; + } + cbm_edge_t *edges = NULL; + int n = 0; + if (cbm_store_find_edges_by_source_type(store, id, "INHERITS", &edges, &n) != CBM_STORE_OK) { + return -1; + } + size_t wl = strlen(want_qn_suffix); + for (int i = 0; i < n; i++) { + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(store, edges[i].target_id, &tgt) == CBM_STORE_OK && + tgt.qualified_name) { + size_t ql = strlen(tgt.qualified_name); + if (ql >= wl && strcmp(tgt.qualified_name + ql - wl, want_qn_suffix) == 0) { + (*to_want)++; + } + } + cbm_node_free_fields(&tgt); + } + cbm_store_free_edges(edges, n); + return n; +} + +/* Count one exact method-to-method OVERRIDE in this fixture's testcases.py. + * Missing endpoints or a failed edge query are fixture failures, not zero. */ +static int ei_testcases_override(cbm_store_t *store, const char *project, const char *source, + const char *target) { + char source_qn[512]; + char target_qn[512]; + snprintf(source_qn, sizeof(source_qn), "%s.pkg.test.testcases.%s", project, source); + snprintf(target_qn, sizeof(target_qn), "%s.pkg.test.testcases.%s", project, target); + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + int found_src = cbm_store_find_node_by_qn(store, project, source_qn, &src); + int found_tgt = cbm_store_find_node_by_qn(store, project, target_qn, &tgt); + int matches = -1; + if (found_src == CBM_STORE_OK && found_tgt == CBM_STORE_OK) { + cbm_edge_t *edges = NULL; + int count = 0; + if (cbm_store_find_edges_by_source_type(store, src.id, "OVERRIDE", &edges, &count) == + CBM_STORE_OK) { + matches = 0; + for (int i = 0; i < count; i++) { + matches += edges[i].target_id == tgt.id; + } + } + cbm_store_free_edges(edges, count); + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + return matches; +} + +/* The django shape (django/test/testcases.py): `class SimpleTestCase( + * unittest.TestCase)` names the STDLIB base through `import unittest`, an + * external module. The registry fell through to a short-name strategy and + * bound it to the project's own `TestCase` -- which itself descends from + * SimpleTestCase, so the graph grew an inheritance CYCLE + * (SimpleTestCase -> TestCase -> TransactionTestCase -> SimpleTestCase) and a + * fabricated OVERRIDE (SimpleTestCase.setUp -> TestCase.setUp). Same family + * as #2127: a name bound by an external import can never denote a same-named + * project symbol. Controls: the project bases (same-module, relative + * `from .testcases import TestCase`, dotted `pkg.test.models.Base` through a + * project `import`) keep their edge. `pad` > MIN_FILES_FOR_PARALLEL(50) runs + * the parallel resolver; 0 the sequential one. */ +static int ei_py_external_base_case(int pad) { + enum { EI_BASE_FIXED = 7, EI_BASE_MAX = EI_BASE_FIXED + 64 }; + static char names[EI_BASE_MAX][32]; + EILangFile f[EI_BASE_MAX]; + int n = 0; + f[n++] = (EILangFile){"pkg/__init__.py", ""}; + f[n++] = (EILangFile){"pkg/test/__init__.py", ""}; + f[n++] = (EILangFile){"pkg/test/testcases.py", "import unittest\n\n\n" + "class SimpleTestCase(unittest.TestCase):\n" + " def setUp(self):\n pass\n\n\n" + "class TransactionTestCase(SimpleTestCase):\n" + " pass\n\n\n" + "class TestCase(TransactionTestCase):\n" + " def setUp(self):\n pass\n"}; + f[n++] = (EILangFile){"pkg/test/models.py", "class Base:\n pass\n"}; + f[n++] = (EILangFile){"pkg/test/suite.py", "import pkg.test.models\n" + "from .testcases import TestCase\n\n\n" + "class RelativeUser(TestCase):\n pass\n\n\n" + "class DottedLocal(pkg.test.models.Base):\n" + " pass\n"}; + f[n++] = (EILangFile){"pkg/test/stdbare.py", "from unittest import TestCase\n\n\n" + "class BareStd(TestCase):\n pass\n"}; + f[n++] = (EILangFile){"pkg/test/stdalias.py", "import unittest as ut\n\n\n" + "class AliasStd(ut.TestCase):\n pass\n"}; + for (int i = 0; i < pad && n < EI_BASE_MAX; i++) { + snprintf(names[n], sizeof(names[n]), "pad/mod_%02d.py", i); + f[n] = (EILangFile){names[n], "def filler():\n return 0\n"}; + n++; + } + EILangProj lp; + cbm_store_t *store = ei_index_files(&lp, f, n); + int w = 0; + int simple = store ? ei_inherits_out(store, lp.project, "SimpleTestCase", ".TestCase", &w) : -1; + int bare = store ? ei_inherits_out(store, lp.project, "BareStd", ".TestCase", &w) : -1; + int alias = store ? ei_inherits_out(store, lp.project, "AliasStd", ".TestCase", &w) : -1; + int w_tx = 0; + int w_tc = 0; + int w_rel = 0; + int w_dot = 0; + int tx = store ? ei_inherits_out(store, lp.project, "TransactionTestCase", + "testcases.SimpleTestCase", &w_tx) + : -1; + int tc = store ? ei_inherits_out(store, lp.project, "TestCase", "testcases.TransactionTestCase", + &w_tc) + : -1; + int rel = store + ? ei_inherits_out(store, lp.project, "RelativeUser", "testcases.TestCase", &w_rel) + : -1; + int dot = store ? ei_inherits_out(store, lp.project, "DottedLocal", "models.Base", &w_dot) : -1; + int overrides = store ? cbm_store_count_edges_by_type(store, lp.project, "OVERRIDE") : -1; + /* #1278: TransactionTestCase declares no setUp. The nearest declaring + * project ancestor of TestCase.setUp is SimpleTestCase.setUp. The missing + * external INHERITS edge must still forbid the reverse override. */ + int forward = store ? ei_testcases_override(store, lp.project, "TestCase.setUp", + "SimpleTestCase.setUp") + : -1; + int reverse = store ? ei_testcases_override(store, lp.project, "SimpleTestCase.setUp", + "TestCase.setUp") + : -1; + int ok = simple == 0 && bare == 0 && alias == 0 && tx == 1 && w_tx == 1 && tc == 1 && + w_tc == 1 && rel == 1 && w_rel == 1 && dot == 1 && w_dot == 1 && overrides == 1 && + forward == 1 && reverse == 0; + if (!ok) { + fprintf(stderr, + " [py-ext-base pad=%d] external bases SimpleTestCase=%d BareStd=%d AliasStd=%d " + "(want 0/0/0); project bases Transaction=%d/%d TestCase=%d/%d Relative=%d/%d " + "Dotted=%d/%d (want 1/1 each); OVERRIDE=%d (want 1); " + "forward=%d reverse=%d (want 1/0)\n", + pad, simple, bare, alias, tx, w_tx, tc, w_tc, rel, w_rel, dot, w_dot, overrides, + forward, reverse); + } + ei_cleanup(&lp, store); + return ok; +} + +TEST(ei_py_external_base_never_inherits_project_class) { + /* Both legs run before asserting so a failure diagnoses both drivers. */ + int sequential_ok = ei_py_external_base_case(0); + int parallel_ok = ei_py_external_base_case(60); + ASSERT_TRUE(sequential_ok); + ASSERT_TRUE(parallel_ok); + PASS(); +} + /* C++: header include should resolve to the header file node, not the same-stem * source node. Also exercises angle-bracket include resolution. */ TEST(ei_cpp_header_include_targets_header_file) { @@ -1279,6 +1447,7 @@ SUITE(edge_imports) { RUN_TEST(ei_go_two_consumers_same_package); RUN_TEST(ei_go_import_never_binds_symbol); RUN_TEST(ei_py_external_import_never_binds_project_symbol); + RUN_TEST(ei_py_external_base_never_inherits_project_class); RUN_TEST(ei_cpp_header_include_targets_header_file); /* ── RED REPRODUCTIONS — Rust (expected to FAIL until pipeline fixed) ── */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9e26829e2..fb47d3a48 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8305,6 +8305,316 @@ TEST(python_crossfile_typed_field_calls_issue1277_parallel) { return check_python_crossfile_typed_field_calls_issue1277(true); } +/* Bind the external-import veto specifically to #1277's typed-field fold. + * No base classes or CALLS edges participate: a name-only call fallback + * cannot turn this metadata assertion into a false positive. The unguarded + * registry deliberately has a same-module TestCase candidate in both cases. */ +static int python_typed_field_scope_case(bool external) { + char source[512]; + snprintf(source, sizeof(source), + "import unittest\n\n" + "class TestCase:\n pass\n\n" + "class Holder:\n field: %s\n", + external ? "unittest.TestCase" : "TestCase"); + CBMFileResult *result = cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, + "proj", "models.py", 0, NULL, NULL); + cbm_registry_t *reg = cbm_registry_new(); + cbm_gbuf_t *gbuf = cbm_gbuf_new("proj", "/unused"); + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef *defs = NULL; + char *modules[1] = {NULL}; + int ok = 0; + if (result && reg && gbuf && result->field_types.count == 1) { + for (int i = 0; i < result->defs.count; i++) { + const CBMDefinition *def = &result->defs.items[i]; + cbm_registry_add(reg, def->name, def->qualified_name, def->label); + } + const char **keys = NULL; + const char **vals = NULL; + int import_count = 0; + cbm_pxc_build_import_map(gbuf, "proj", "models.py", CBM_LANG_PYTHON, result, &keys, + &vals, &import_count); + cbm_resolution_t raw = cbm_registry_resolve( + reg, external ? "unittest.TestCase" : "TestCase", "proj.models", keys, vals, + import_count); + bool candidate = raw.qualified_name && raw.strategy && + strcmp(raw.qualified_name, "proj.models.TestCase") == 0 && + strcmp(raw.strategy, "same_module") == 0; + cbm_pxc_free_import_map(keys, vals, import_count); + cbm_pipeline_ctx_t ctx = {.project_name = "proj", .gbuf = gbuf, .registry = reg}; + cbm_file_info_t files[1] = {{.rel_path = "models.py", .language = CBM_LANG_PYTHON}}; + CBMFileResult *cache[1] = {result}; + int count = 0; + defs = cbm_pxc_collect_all_defs(&ctx, &arena, cache, files, 1, "proj", modules, &count, + NULL); + int holders = 0; + bool field_ok = false; + for (int i = 0; defs && i < count; i++) { + if (defs[i].qualified_name && + strcmp(defs[i].qualified_name, "proj.models.Holder") == 0) { + holders++; + field_ok = external ? !defs[i].field_defs + : defs[i].field_defs && + strcmp(defs[i].field_defs, + "field:proj.models.TestCase") == 0; + } + } + ok = candidate && holders == 1 && field_ok; + if (!ok) { + fprintf(stderr, " [py-field-scope external=%d] candidate=%d holders=%d field_ok=%d\n", + external, candidate, holders, field_ok); + } + } + free(defs); + free(modules[0]); + cbm_arena_destroy(&arena); + cbm_gbuf_free(gbuf); + cbm_registry_free(reg); + cbm_free_result(result); + return ok; +} + +TEST(python_typed_field_external_import_stays_unresolved) { + ASSERT_TRUE(python_typed_field_scope_case(true)); + PASS(); +} + +TEST(python_typed_field_project_class_resolves) { + ASSERT_TRUE(python_typed_field_scope_case(false)); + PASS(); +} + +/* ── Explicit OVERRIDE through non-redeclaring ancestors (#1278) ── */ + +static int64_t ovr_class(cbm_gbuf_t *gb, const char *name) { + char qn[128]; + snprintf(qn, sizeof(qn), "m.%s", name); + return cbm_gbuf_upsert_node(gb, "Class", name, qn, "m.py", 1, 50, "{}"); +} + +static int64_t ovr_method(cbm_gbuf_t *gb, int64_t cls, const char *cls_name, const char *name) { + char qn[128]; + snprintf(qn, sizeof(qn), "m.%s.%s", cls_name, name); + int64_t id = cbm_gbuf_upsert_node(gb, "Method", name, qn, "m.py", 2, 3, "{}"); + cbm_gbuf_insert_edge(gb, cls, id, "DEFINES_METHOD", "{}"); + return id; +} + +/* OVERRIDE edges leaving `method`; *only_target receives the single target (or 0). */ +static int ovr_out(cbm_gbuf_t *gb, int64_t method, int64_t *only_target) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_source_type(gb, method, "OVERRIDE", &edges, &count); + *only_target = count == 1 ? edges[0]->target_id : 0; + return count; +} + +/* Exactly two different targets, each once, independent of edge order. */ +static bool ovr_two_targets(cbm_gbuf_t *gb, int64_t method, int64_t first, int64_t second) { + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_source_type(gb, method, "OVERRIDE", &edges, &count); + int first_count = 0; + int second_count = 0; + for (int i = 0; i < count; i++) { + first_count += edges[i]->target_id == first; + second_count += edges[i]->target_id == second; + } + return first != second && count == 2 && first_count == 1 && second_count == 1; +} + +TEST(override_explicit_walks_to_nearest_declaring_ancestor) { + cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test"); + ASSERT_NOT_NULL(gb); + + /* Contract: Processor.process */ + int64_t processor = ovr_class(gb, "Processor"); + int64_t proc_process = ovr_method(gb, processor, "Processor", "process"); + + /* Direct control: DirectProcessor(Processor) */ + int64_t direct = ovr_class(gb, "DirectProcessor"); + int64_t direct_process = ovr_method(gb, direct, "DirectProcessor", "process"); + cbm_gbuf_insert_edge(gb, direct, processor, "INHERITS", "{}"); + + /* The #1278 shape: LeafProcessor(IntermediateProcessor(Processor)), the + * intermediate declares nothing. `extra` exists on no ancestor. */ + int64_t mid = ovr_class(gb, "IntermediateProcessor"); + int64_t leaf = ovr_class(gb, "LeafProcessor"); + int64_t leaf_process = ovr_method(gb, leaf, "LeafProcessor", "process"); + int64_t leaf_extra = ovr_method(gb, leaf, "LeafProcessor", "extra"); + cbm_gbuf_insert_edge(gb, mid, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, leaf, mid, "INHERITS", "{}"); + + /* Nearest wins: Deep(Redeclaring(Processor)), Redeclaring declares process. */ + int64_t redecl = ovr_class(gb, "Redeclaring"); + int64_t redecl_process = ovr_method(gb, redecl, "Redeclaring", "process"); + int64_t deep = ovr_class(gb, "Deep"); + int64_t deep_process = ovr_method(gb, deep, "Deep", "process"); + cbm_gbuf_insert_edge(gb, redecl, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, deep, redecl, "INHERITS", "{}"); + + /* Diamond: Both(Left, Right), Left and Right each inherit Processor + * without redeclaring -> exactly one edge, no duplicate. */ + int64_t left = ovr_class(gb, "Left"); + int64_t right = ovr_class(gb, "Right"); + int64_t both = ovr_class(gb, "Both"); + int64_t both_process = ovr_method(gb, both, "Both", "process"); + cbm_gbuf_insert_edge(gb, left, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, right, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, both, left, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, both, right, "INHERITS", "{}"); + + /* Equal nearest depth: two declaring ancestors behind empty direct + * parents. Both declarations bind, while the deeper Processor does not. + * Both explicit graph relationships share the nearest declaring depth. */ + int64_t tie_left = ovr_class(gb, "TieLeft"); + int64_t tie_right = ovr_class(gb, "TieRight"); + int64_t tie_left_process = ovr_method(gb, tie_left, "TieLeft", "process"); + int64_t tie_right_process = ovr_method(gb, tie_right, "TieRight", "process"); + int64_t tie_left_mid = ovr_class(gb, "TieLeftMid"); + int64_t tie_right_mid = ovr_class(gb, "TieRightMid"); + int64_t tied = ovr_class(gb, "Tied"); + int64_t tied_process = ovr_method(gb, tied, "Tied", "process"); + cbm_gbuf_insert_edge(gb, tie_left, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, tie_right, processor, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, tie_left_mid, tie_left, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, tie_right_mid, tie_right, "IMPLEMENTS", "{}"); + cbm_gbuf_insert_edge(gb, tied, tie_right_mid, "IMPLEMENTS", "{}"); + cbm_gbuf_insert_edge(gb, tied, tie_left_mid, "INHERITS", "{}"); + + /* The separate Go implicit-satisfaction pass owns .go classes. */ + int64_t go_type = + cbm_gbuf_upsert_node(gb, "Struct", "GoType", "m.GoType", "m.go", 1, 5, "{}"); + int64_t go_process = ovr_method(gb, go_type, "GoType", "process"); + cbm_gbuf_insert_edge(gb, go_type, processor, "IMPLEMENTS", "{}"); + + /* Malformed hierarchy: CycA <-> CycB plus a self-loop, CycLeaf(CycA). + * The walk must terminate and bind nothing. */ + int64_t cyc_a = ovr_class(gb, "CycA"); + int64_t cyc_b = ovr_class(gb, "CycB"); + int64_t cyc_leaf = ovr_class(gb, "CycLeaf"); + int64_t cyc_process = ovr_method(gb, cyc_leaf, "CycLeaf", "process"); + cbm_gbuf_insert_edge(gb, cyc_a, cyc_b, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, cyc_b, cyc_a, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, cyc_a, cyc_a, "INHERITS", "{}"); + cbm_gbuf_insert_edge(gb, cyc_leaf, cyc_a, "INHERITS", "{}"); + + atomic_int cancelled = 0; + cbm_pipeline_ctx_t ctx = { + .project_name = "test-proj", + .repo_path = "/tmp/test", + .gbuf = gb, + .cancelled = &cancelled, + }; + (void)cbm_pipeline_override_explicit(&ctx); + + int64_t target = 0; + bool direct_ok = ovr_out(gb, direct_process, &target) == 1 && target == proc_process; + bool indirect_ok = ovr_out(gb, leaf_process, &target) == 1 && target == proc_process; + bool extra_absent = ovr_out(gb, leaf_extra, &target) == 0; + bool nearest_ok = ovr_out(gb, deep_process, &target) == 1 && target == redecl_process; + bool redecl_ok = ovr_out(gb, redecl_process, &target) == 1 && target == proc_process; + bool diamond_ok = ovr_out(gb, both_process, &target) == 1 && target == proc_process; + bool ties_ok = ovr_two_targets(gb, tied_process, tie_left_process, tie_right_process); + bool go_absent = ovr_out(gb, go_process, &target) == 0; + bool cycle_absent = ovr_out(gb, cyc_process, &target) == 0; + int first_count = cbm_gbuf_edge_count_by_type(gb, "OVERRIDE"); + + /* Idempotent: a second run adds no edges. */ + (void)cbm_pipeline_override_explicit(&ctx); + int second_count = cbm_gbuf_edge_count_by_type(gb, "OVERRIDE"); + bool second_ties_ok = ovr_two_targets(gb, tied_process, tie_left_process, tie_right_process); + + cbm_gbuf_free(gb); + ASSERT_TRUE(direct_ok); + ASSERT_TRUE(indirect_ok); + ASSERT_TRUE(extra_absent); + ASSERT_TRUE(nearest_ok); + ASSERT_TRUE(redecl_ok); + ASSERT_TRUE(diamond_ok); + ASSERT_TRUE(ties_ok); + ASSERT_TRUE(go_absent); + ASSERT_TRUE(cycle_absent); + ASSERT_EQ(first_count, 9); + ASSERT_EQ(second_count, 9); + ASSERT_TRUE(second_ties_ok); + PASS(); +} + +/* OVERRIDE edges from a method whose QN ends with src_suffix to one whose QN + * ends with tgt_suffix, in the stored graph. */ +static int count_override_edges(cbm_store_t *s, const char *project, const char *src_suffix, + const char *tgt_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "OVERRIDE", &edges, &edge_count); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + size_t ss = strlen(src_suffix); + size_t ts = strlen(tgt_suffix); + if (sl >= ss && tl >= ts && strcmp(src.qualified_name + sl - ss, src_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, tgt_suffix) == 0) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return hits; +} + +TEST(override_python_through_intermediate_class) { + /* The #1278 reproduction, end to end through the full pipeline. */ + const char *py = "from abc import ABC, abstractmethod\n\n" + "class Processor(ABC):\n" + " @abstractmethod\n" + " def process(self) -> str:\n" + " raise NotImplementedError\n\n" + "class IntermediateProcessor(Processor):\n" + " pass\n\n" + "class LeafProcessor(IntermediateProcessor):\n" + " def process(self) -> str:\n" + " return \"leaf\"\n\n" + "class DirectProcessor(Processor):\n" + " def process(self) -> str:\n" + " return \"direct\"\n\n" + "def run(processor: Processor) -> str:\n" + " return processor.process()\n"; + if (setup_usages_repo("procs.py", py, NULL, NULL) != 0) { + FAIL("failed to create temp dir"); + } + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_override.db", g_usages_tmpdir); + cbm_pipeline_t *p = cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + + int direct = count_override_edges(s, project, "DirectProcessor.process", ".Processor.process"); + int leaf = count_override_edges(s, project, "LeafProcessor.process", ".Processor.process"); + int all = count_override_edges(s, project, "", ""); + + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_usages_repo(); + + ASSERT_EQ(direct, 1); /* control */ + ASSERT_EQ(leaf, 1); /* #1278 */ + ASSERT_EQ(all, 2); /* nothing else, no duplicates */ + PASS(); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -16517,8 +16827,12 @@ SUITE(pipeline) { RUN_TEST(implements_creates_override); RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ + RUN_TEST(override_explicit_walks_to_nearest_declaring_ancestor); + RUN_TEST(override_python_through_intermediate_class); RUN_TEST(python_crossfile_typed_field_calls_issue1277); RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); + RUN_TEST(python_typed_field_external_import_stays_unresolved); + RUN_TEST(python_typed_field_project_class_resolves); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); diff --git a/tests/test_registry.c b/tests/test_registry.c index 278e97d9b..fd9b547ee 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1230,6 +1230,42 @@ TEST(python_import_binding_contradicts_only_foreign_chains) { PASS(); } +/* Base-class twin: the django shapes. Every spelling of the stdlib base must + * contradict the same-named project class; project bases never do, and the + * gate is Python-only. */ +TEST(python_external_base_contradicts_django_shapes) { + CBMImport items[] = { + {.local_name = "unittest", .module_path = "unittest"}, + {.local_name = "ut", .module_path = "unittest"}, + {.local_name = "TestCase", .module_path = "unittest.TestCase"}, + {.local_name = "pkg", .module_path = "pkg.test.models"}, + }; + CBMImportArray imps = {.items = items, .count = 4, .cap = 4}; + const char *project_tc = "proj.django.test.testcases.TestCase"; + ASSERT_TRUE(cbm_python_external_base_contradicts(CBM_LANG_PYTHON, &imps, "unittest.TestCase", + project_tc, NULL, NULL, NULL)); + ASSERT_TRUE(cbm_python_external_base_contradicts(CBM_LANG_PYTHON, &imps, "TestCase", project_tc, + NULL, NULL, NULL)); + /* `import unittest as ut`: `ut.TestCase` spells `unittest.TestCase`. */ + ASSERT_TRUE(cbm_python_external_base_contradicts(CBM_LANG_PYTHON, &imps, "ut.TestCase", + project_tc, NULL, NULL, NULL)); + /* A target that does spell the chain (vendored/src-layout unittest). */ + ASSERT_FALSE(cbm_python_external_base_contradicts( + CBM_LANG_PYTHON, &imps, "ut.TestCase", "proj.src.unittest.TestCase", NULL, NULL, NULL)); + /* Dotted project base through `import pkg.test.models`. */ + ASSERT_FALSE( + cbm_python_external_base_contradicts(CBM_LANG_PYTHON, &imps, "pkg.test.models.Base", + "proj.pkg.test.models.Base", NULL, NULL, NULL)); + /* Not import-bound: a same-module base is never judged. */ + ASSERT_FALSE(cbm_python_external_base_contradicts(CBM_LANG_PYTHON, &imps, "SimpleTestCase", + "proj.django.test.testcases.SimpleTestCase", + NULL, NULL, NULL)); + /* Python only. */ + ASSERT_FALSE(cbm_python_external_base_contradicts(CBM_LANG_JAVA, &imps, "unittest.TestCase", + project_tc, NULL, NULL, NULL)); + PASS(); +} + TEST(weak_call_guards_share_one_drop_list) { /* The member guard and the local-binding guard must agree on what "weak" * means. They share a single static predicate for exactly this reason; if @@ -1371,5 +1407,6 @@ SUITE(registry) { RUN_TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies); RUN_TEST(import_binding_suppress_drops_only_weak_contradicted_calls); RUN_TEST(python_import_binding_contradicts_only_foreign_chains); + RUN_TEST(python_external_base_contradicts_django_shapes); RUN_TEST(weak_call_guards_share_one_drop_list); } From fa258d342734d56e88180017bf0d92222813645d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 25 Sep 2026 18:37:16 +0200 Subject: [PATCH 15/34] fix(extraction): parse every file as if its last line were terminated (#2078) A file whose last byte is not "\n" leaves the construct on its final line unterminated for every grammar that treats the line ending as part of that construct. #1610/#1746 already excused the mild form, a ZERO-WIDTH MISSING newline at EOF. The severe form is a WIDTH-BEARING error that cannot be excused. In tree-sitter-markdown an opening fence, a bare list marker or an ATX heading as the last bytes costs the last line (`text\n` + fence -> parse_partial 2-2) or the whole file (a lone fence, `- ` -> parse_unusable). An unterminated one-line heading was lost silently: the root is ERROR, there is no Section and no flag. A Makefile whose last recipe line is unterminated lost the recipe. The same bytes plus one "\n" parse clean. The reporter confirmed this on 21 real files (43 -> 23 flagged after appending newlines). Supply the newline rather than excuse its absence. cbm_parse_source() feeds the parser one virtual "\n" at offset source_len (through the TSInput read callback, so the caller's buffer is never copied or written) and then applies one ts_tree_edit that deletes that byte again. tree-sitter's edit arithmetic clamps every node that reached into it back to the real end, bytes and row/column points both. So def line ranges, node text, call byte spans, parse-coverage ranges and the retained tree all see only real bytes and real lines. Empty files and files that already end in "\n" are parsed exactly as before. Every whole-file parse goes through the helper: the extraction parse, and the whole-file fallback re-parses in the C/C++, C#, Go, Java, Kotlin, PHP, Python, Rust and TS LSP passes. That keeps a spilled result (no retained tree) consistent with a cached one. Sub-range and synthetic parses (preprocessed C, Rust wrappers, Kotlin patches, injected Jinja/imports ranges) are unchanged. The two #1610/#1746 guards that pinned "an unterminated Makefile recipe is a width-bearing loss" now use a construct that is broken with or without a newline (an unclosed Python tuple at EOF). The recipe is no longer lost, so flagging it would be false. The new makefile_unterminated_recipe_is_parsed_issue2078 test pins that. Proof (this repo, vendored grammars excluded, before/after binaries): - unchanged tree, and the same tree with the final newline stripped from all 1,220 files: nodes, edges and parse_partial lists byte-identical before vs after; - EOF-class corpus (the repo plus each .md with a fence, list marker, heading or opening fence appended without a newline, plus an unterminated Makefile): parse_partial 122 -> 77, exactly the unchanged repo's list. The only graph deltas are the 45 dropped ::missed shadow File rows (+7 Folders, 52 CONTAINS edges) and 22 Section docstrings that now include their real last line. The other four constructs in #2078 (HTML bare `&`, bash heredoc + redirect + pipe, bash `$((10#$n))`, markdown empty `||` cell) are upstream grammar gaps. Reports for them are drafted separately. Refs #2078 (the EOF class; the remaining triggers are upstream grammar gaps) Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 84 ++++++++-- internal/cbm/cbm.h | 8 + internal/cbm/lsp/c_lsp.c | 4 +- internal/cbm/lsp/cs_lsp.c | 10 +- internal/cbm/lsp/go_lsp.c | 4 +- internal/cbm/lsp/java_lsp.c | 4 +- internal/cbm/lsp/kotlin_lsp.c | 2 +- internal/cbm/lsp/php_lsp.c | 2 +- internal/cbm/lsp/py_lsp.c | 4 +- internal/cbm/lsp/rust_lsp.c | 4 +- internal/cbm/lsp/ts_lsp.c | 4 +- tests/test_parse_coverage.c | 285 +++++++++++++++++++++++++++++++++- 12 files changed, 376 insertions(+), 39 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 4d01d0c69..035285c30 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -278,6 +278,8 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) { typedef struct { const char *string; uint32_t length; + /* #2078: serve one "\n" at offset `length` -- see cbm_parse_source. */ + bool virtual_newline; } CBMStringInput; static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, @@ -285,6 +287,10 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, (void)point; CBMStringInput *self = (CBMStringInput *)payload; if (byte >= self->length) { + if (self->virtual_newline && byte == self->length) { + *bytes_read = 1; + return "\n"; + } *bytes_read = 0; return ""; } @@ -292,6 +298,60 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point, return self->string + byte; } +/* Where the parser stands after reading all `len` real bytes. */ +static TSPoint cbm_source_end_point(const char *source, uint32_t len) { + TSPoint end = {0, 0}; + const char *p = source; + const char *stop = source + len; + const char *nl; + while (p < stop && (nl = memchr(p, '\n', (size_t)(stop - p))) != NULL) { + end.row++; + p = nl + 1; + } + end.column = (uint32_t)(stop - p); + return end; +} + +/* #2078: parse every file as if its last line were terminated. + * + * Many grammars treat the line terminator as part of the construct it ends -- + * a markdown fence, heading or list marker, a Makefile recipe, a Dockerfile + * instruction. A file whose last byte is not "\n" leaves that construct + * unterminated: at best a zero-width MISSING token (#1610/#1746 excuse those at + * EOF), at worst a WIDTH-BEARING error that costs the whole last line or, for a + * one-line file, the whole file, and sometimes a silent loss (an unterminated + * markdown heading produced no Section and no flag). The same bytes plus one + * "\n" parse clean, so the absent newline is supplied instead of excused. + * + * The parser reads one virtual "\n" past EOF; afterwards a single tree edit + * deletes it again. tree-sitter's own edit arithmetic then clamps every node + * that reached into the virtual byte back to the real end -- byte offsets AND + * row/column points -- so every consumer (def line ranges, node text, call + * byte spans, parse-coverage ranges, the retained tree the LSP passes reuse) + * sees only real bytes and real lines; nothing downstream needs to know. The + * caller's buffer is never copied or written. + * + * A file that is empty or already ends in "\n" is parsed exactly as before. */ +TSTree *cbm_parse_source(TSParser *parser, const char *source, uint32_t source_len, + TSParseOptions opts) { + CBMStringInput input = {source, source_len, source_len > 0 && source[source_len - 1] != '\n'}; + TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL}; + TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); + if (tree && input.virtual_newline) { + TSPoint end = cbm_source_end_point(source, source_len); + TSInputEdit drop_virtual_newline = { + .start_byte = source_len, + .old_end_byte = source_len + 1, + .new_end_byte = source_len, + .start_point = end, + .old_end_point = {end.row + 1, 0}, + .new_end_point = end, + }; + ts_tree_edit(tree, &drop_virtual_newline); + } + return tree; +} + // --- Parse timeout callback --- /* Budget for the tree-sitter progress callback. The PRIMARY gate is per-thread @@ -1261,7 +1321,11 @@ static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) { * #1746: the Dockerfile grammar places that zero-width missing newline before * trailing whitespace rather than at raw EOF. Preserve the broad exact-EOF * rule above; only extend it past blanks when the missing token is specifically - * a newline. */ + * a newline. + * + * #2078: cbm_parse_source now supplies the absent final newline, so a MISSING + * newline at EOF no longer arises from it; the rule stays for the other + * zero-width terminators a grammar can leave at EOF. */ static bool cbm_is_blank_not_newline(char c) { return c == ' ' || c == '\t' || c == '\v' || c == '\f' || c == '\r'; } @@ -2221,15 +2285,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C uint64_t t0 = now_ns(); - // Build string input + timeout options for parse_with_options - CBMStringInput str_input = {source, (uint32_t)source_len}; - TSInput ts_input = { - &str_input, - cbm_string_read, - TSInputEncodingUTF8, - NULL, - }; - + // Timeout options for parse_with_options TSParseOptions opts = {0}; CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable uint64_t budget_ns = 0; @@ -2257,12 +2313,14 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C /* #1735: a SQL data dump's literal-only INSERT rows carry no graph content * but dominate its parse. Keep them out through included ranges; offsets * and positions of everything kept are unchanged. The parser is - * thread-local and reused, so the ranges are cleared right after. */ + * thread-local and reused, so the ranges are cleared right after. The last + * range ends at the real EOF, so the virtual final newline cbm_parse_source + * supplies (#2078) stays outside a ranged parse. */ CBMSqlKeptRanges sql_kept = {NULL, 0}; bool sql_ranged = language == CBM_LANG_SQL && cbm_sql_values_exclusion_on(rel_path) && cbm_sql_values_kept_ranges(source, (uint32_t)source_len, &sql_kept) && ts_parser_set_included_ranges(parser, sql_kept.items, sql_kept.count); - TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); + TSTree *tree = cbm_parse_source(parser, source, (uint32_t)source_len, opts); if (sql_ranged) { (void)ts_parser_set_included_ranges(parser, NULL, 0); } @@ -2465,7 +2523,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C TSParser *pp_parser = get_thread_parser(ts_lang, language); if (pp_parser) { ts_parser_reset(pp_parser); - CBMStringInput pp_input = {expanded, (uint32_t)expanded_len}; + CBMStringInput pp_input = {expanded, (uint32_t)expanded_len, false}; TSInput pp_ts_input = { &pp_input, cbm_string_read, diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index dc89a6b2d..3bd310d8b 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -819,6 +819,14 @@ void cbm_work_arena_keep_begin(void); /* Free the compaction scratch this thread kept (cbm_work_arena_release calls it). */ void cbm_result_compact_release_thread(void); +/* Parse one whole file as if its last line ended with "\n" (#2078). The + * parser sees the source plus one virtual newline when the last byte is not + * already one; the returned tree is then clamped back to `source_len`, so no + * node range, point or text reaches past the real bytes. Every whole-file parse + * goes through here, so a retained tree and a fallback re-parse agree. */ +TSTree *cbm_parse_source(TSParser *parser, const char *source, uint32_t source_len, + TSParseOptions opts); + // Extract all data from one file. Caller must call cbm_free_result(). // source must remain valid for the duration of the call. // timeout_micros: per-file tree-sitter parse budget in microseconds of the diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 22d630020..73141b0f6 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -6101,7 +6101,7 @@ void cbm_run_c_lsp_cross_with_registry(CBMArena *arena, const char *source, int return; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); owns_tree = true; if (!tree) @@ -6151,7 +6151,7 @@ void cbm_run_c_lsp_cross(CBMArena *arena, const char *source, int source_len, co return; const TSLanguage *ts_lang = cpp_mode ? tree_sitter_cpp() : tree_sitter_c(); ts_parser_set_language(parser, ts_lang); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); owns_tree = true; if (!tree) diff --git a/internal/cbm/lsp/cs_lsp.c b/internal/cbm/lsp/cs_lsp.c index fc3b548ef..749b126bf 100644 --- a/internal/cbm/lsp/cs_lsp.c +++ b/internal/cbm/lsp/cs_lsp.c @@ -3763,8 +3763,9 @@ void cbm_run_cs_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!parser) return; ts_parser_set_language(parser, tree_sitter_c_sharp()); - tree = ts_parser_parse_string( - parser, NULL, source, source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source)); + tree = cbm_parse_source(parser, source, + source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source), + (TSParseOptions){0}); ts_parser_delete(parser); owns = true; } @@ -3805,8 +3806,9 @@ void cbm_run_cs_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!parser) return; ts_parser_set_language(parser, tree_sitter_c_sharp()); - tree = ts_parser_parse_string( - parser, NULL, source, source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source)); + tree = cbm_parse_source(parser, source, + source_len > 0 ? (uint32_t)source_len : (uint32_t)strlen(source), + (TSParseOptions){0}); ts_parser_delete(parser); owns = true; } diff --git a/internal/cbm/lsp/go_lsp.c b/internal/cbm/lsp/go_lsp.c index ba7df163b..58c151a53 100644 --- a/internal/cbm/lsp/go_lsp.c +++ b/internal/cbm/lsp/go_lsp.c @@ -3224,7 +3224,7 @@ void cbm_run_go_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!parser) return; ts_parser_set_language(parser, tree_sitter_go()); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); @@ -3630,7 +3630,7 @@ void cbm_run_go_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!parser) return; ts_parser_set_language(parser, tree_sitter_go()); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); diff --git a/internal/cbm/lsp/java_lsp.c b/internal/cbm/lsp/java_lsp.c index 9a79783ff..2b518d890 100644 --- a/internal/cbm/lsp/java_lsp.c +++ b/internal/cbm/lsp/java_lsp.c @@ -3931,7 +3931,7 @@ void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result return; } ts_parser_set_language(parser, tree_sitter_java()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); owns_tree = true; } @@ -3985,7 +3985,7 @@ void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!tree) { TSParser *parser = ts_parser_new(); ts_parser_set_language(parser, tree_sitter_java()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); owns_tree = true; } diff --git a/internal/cbm/lsp/kotlin_lsp.c b/internal/cbm/lsp/kotlin_lsp.c index 0ca49afd5..cc6be1680 100644 --- a/internal/cbm/lsp/kotlin_lsp.c +++ b/internal/cbm/lsp/kotlin_lsp.c @@ -5415,7 +5415,7 @@ void cbm_run_kotlin_lsp_cross(CBMArena *arena, const char *source, int source_le return; } ts_parser_set_language(parser, tree_sitter_kotlin()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); owns_tree = true; } diff --git a/internal/cbm/lsp/php_lsp.c b/internal/cbm/lsp/php_lsp.c index 4caf5ba0f..81347a94c 100644 --- a/internal/cbm/lsp/php_lsp.c +++ b/internal/cbm/lsp/php_lsp.c @@ -4498,7 +4498,7 @@ void cbm_run_php_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!parser) return; ts_parser_set_language(parser, tree_sitter_php_only()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 70de5dce7..2b47fbf23 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -5110,7 +5110,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, if (!parser) return; ts_parser_set_language(parser, tree_sitter_python()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); @@ -5191,7 +5191,7 @@ void cbm_run_py_lsp_cross_with_registry(CBMArena *arena, const char *source, int if (!parser) return; ts_parser_set_language(parser, tree_sitter_python()); - tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); diff --git a/internal/cbm/lsp/rust_lsp.c b/internal/cbm/lsp/rust_lsp.c index 68fb444a6..44680cb69 100644 --- a/internal/cbm/lsp/rust_lsp.c +++ b/internal/cbm/lsp/rust_lsp.c @@ -6377,7 +6377,7 @@ void cbm_run_rust_lsp_cross_with_registry(CBMArena *arena, const char *source, i if (!parser) return; ts_parser_set_language(parser, tree_sitter_rust()); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); @@ -6412,7 +6412,7 @@ void cbm_run_rust_lsp_cross_with_manifest(CBMArena *arena, const char *source, i if (!parser) return; ts_parser_set_language(parser, tree_sitter_rust()); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); owns_tree = true; if (!tree) { ts_parser_delete(parser); diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 967d06762..c25a58bbe 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -5899,7 +5899,7 @@ void cbm_run_ts_lsp_cross_with_registry(CBMArena *arena, const char *source, int jsx_mode ? (js_mode ? tree_sitter_javascript() : tree_sitter_tsx()) : (js_mode ? tree_sitter_javascript() : tree_sitter_typescript()); ts_parser_set_language(parser, lang); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); if (!tree) return; @@ -6110,7 +6110,7 @@ void cbm_run_ts_lsp_cross(CBMArena *arena, const char *source, int source_len, jsx_mode ? (js_mode ? tree_sitter_javascript() : tree_sitter_tsx()) : (js_mode ? tree_sitter_javascript() : tree_sitter_typescript()); ts_parser_set_language(parser, lang); - tree = ts_parser_parse_string(parser, NULL, source, source_len); + tree = cbm_parse_source(parser, source, (uint32_t)source_len, (TSParseOptions){0}); ts_parser_delete(parser); if (!tree) return; diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c index 09caeaa82..1cc2f8e98 100644 --- a/tests/test_parse_coverage.c +++ b/tests/test_parse_coverage.c @@ -507,11 +507,17 @@ TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610) { } /* GUARD: a MISSING/ERROR node WITH WIDTH at EOF is a genuine loss and must - * still be flagged. A Makefile whose final recipe line lacks its newline really - * does drop the recipe from the tree — cbm's flag is honest there. */ + * still be flagged. The construct here is broken whether or not the line is + * terminated, so neither the zero-width rule nor the virtual final newline + * (#2078) may excuse it. + * + * This guard used a Makefile whose final recipe line lacks its newline, which + * really did drop the recipe from the tree. Since #2078 the parser sees that + * line terminated and the recipe parses, so there is no loss left to report -- + * see makefile_unterminated_recipe_is_parsed_issue2078. */ TEST(width_bearing_error_at_eof_still_flagged_issue1610) { - const char *src = "all:\n\techo hi"; /* no trailing newline; recipe is lost */ - CBMFileResult *r = do_extract(src, CBM_LANG_MAKEFILE, "Makefile"); + const char *src = "def ok():\n return 1\nx = (1,"; /* unclosed at EOF */ + CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "eof.py"); ASSERT_NOT_NULL(r); bool flagged = r->parse_incomplete; cbm_free_result(r); @@ -521,6 +527,22 @@ TEST(width_bearing_error_at_eof_still_flagged_issue1610) { PASS(); } +/* #2078: the recipe the old #1610 guard lost is now parsed -- the file is not + * flagged, with or without a trailing blank. */ +TEST(makefile_unterminated_recipe_is_parsed_issue2078) { + const char *cases[] = {"all:\n\techo hi", "all:\n\techo hi "}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = do_extract(cases[i], CBM_LANG_MAKEFILE, "Makefile"); + ASSERT_NOT_NULL(r); + bool flagged = r->parse_incomplete; + cbm_free_result(r); + if (flagged) { + FAIL("an unterminated final recipe line must parse like a terminated one"); + } + } + PASS(); +} + /* #1838: tree-sitter-perl v1.0.0 rejects a valid line-oriented format and * reports the file as partial. The supported upstream v1.2.1 grammar accepts * the format and preserves declaration extraction beyond its dot terminator. */ @@ -611,11 +633,12 @@ TEST(real_error_before_eof_still_flagged_with_trailing_blank_issue1746) { PASS(); } -/* GUARD: a WIDTH-BEARING loss at EOF stays honest with a blank tail too — the - * Makefile recipe really is dropped, and only zero-width nodes are excused. */ +/* GUARD: a WIDTH-BEARING loss at EOF stays honest with a blank tail too, and + * only zero-width nodes are excused. (Formerly the unterminated Makefile + * recipe, which #2078 now parses -- see the #1610 guard above.) */ TEST(width_bearing_error_at_eof_still_flagged_with_trailing_blank_issue1746) { - const char *src = "all:\n\techo hi "; - CBMFileResult *r = do_extract(src, CBM_LANG_MAKEFILE, "Makefile"); + const char *src = "def ok():\n return 1\nx = (1, "; + CBMFileResult *r = do_extract(src, CBM_LANG_PYTHON, "eof.py"); ASSERT_NOT_NULL(r); bool flagged = r->parse_incomplete; cbm_free_result(r); @@ -1615,6 +1638,246 @@ TEST(cs_malformed_conditional_remains_partial_issue1748) { PASS(); } +/* ── #2078: every file is parsed as if it ended with a newline ─────────────── + * + * Several grammars need a line terminator that a file's last line may simply + * not have. tree-sitter-markdown is the sharpest case: an opening code fence, + * an ATX heading or a bare list marker as the final bytes of a file leaves a + * WIDTH-BEARING ERROR (so the #1610 zero-width rule cannot excuse it), and a + * file that is only "```" came back as a whole-file error. Appending one "\n" + * made every one of these clean -- the reporter proved it on 21 real files. + * + * cbm now feeds the parser one virtual "\n" past EOF when the last byte is not + * already a newline, then clamps the tree back to the real length, so no + * consumer sees a byte, a line or a range that is not in the file. */ +TEST(markdown_unterminated_last_line_is_clean_issue2078) { + const char *cases[] = { + "text\n```", /* opening fence at EOF (upstream PR #262, closed unmerged) */ + "```", /* the whole file is one opening fence */ + "# Top", /* ATX heading */ + "- ", /* bare list marker */ + }; + int flagged = 0; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md"); + ASSERT_NOT_NULL(r); + if (r->parse_incomplete || r->parse_unusable) { + fprintf(stderr, " case %zu flagged: partial=%d unusable=%d ranges=%s\n", i, + r->parse_incomplete, r->parse_unusable, + r->error_ranges ? r->error_ranges : "(none)"); + flagged++; + } + cbm_free_result(r); + } + if (flagged) { + FAIL("a markdown file must not be flagged only because its last line is unterminated"); + } + PASS(); +} + +/* Controls: the same bytes WITH the newline were already clean and stay so. */ +TEST(markdown_terminated_last_line_controls_unchanged_issue2078) { + const char *cases[] = {"text\n```\n", "```\n", "# Top\n", "- \n"}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + CBMFileResult *r = do_extract(cases[i], CBM_LANG_MARKDOWN, "doc.md"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_FALSE(r->parse_unusable); + cbm_free_result(r); + } + PASS(); +} + +/* Everything extraction reports, as one string, so two results compare + * byte-for-byte: defs (every text field and range), calls (exact byte spans), + * imports and the coverage verdict. */ +typedef struct { + char *buf; + size_t cap; + size_t len; +} dump_buf_t; + +static void dump_str(dump_buf_t *b, const char *tag, const char *s) { + int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%s", tag, s ? s : "~"); + if (n > 0 && (size_t)n < b->cap - b->len) { + b->len += (size_t)n; + } +} + +static void dump_num(dump_buf_t *b, const char *tag, long v) { + int n = snprintf(b->buf + b->len, b->cap - b->len, " %s=%ld", tag, v); + if (n > 0 && (size_t)n < b->cap - b->len) { + b->len += (size_t)n; + } +} + +/* The Module def spans the whole file, and its end line follows the line + * convention for a terminated last line (#1967), so it is compared on its own + * rather than through the dump. */ +static char *dump_extraction(const CBMFileResult *r) { + dump_buf_t b = {(char *)calloc(1u << 16, 1), 1u << 16, 0}; + if (!b.buf) { + return NULL; + } + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && strcmp(d->label, "Module") == 0) { + continue; + } + dump_str(&b, "\nD", d->qualified_name); + dump_str(&b, "name", d->name); + dump_str(&b, "label", d->label); + dump_num(&b, "start", d->start_line); + dump_num(&b, "end", d->end_line); + dump_num(&b, "lines", d->lines); + dump_num(&b, "cx", d->complexity); + dump_num(&b, "params", d->param_count); + dump_str(&b, "sig", d->signature); + dump_str(&b, "ret", d->return_type); + dump_str(&b, "doc", d->docstring); + dump_str(&b, "prof", d->structural_profile); + dump_str(&b, "tok", d->body_tokens); + } + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + dump_str(&b, "\nC", c->callee_name); + dump_str(&b, "in", c->enclosing_func_qn); + dump_num(&b, "line", c->start_line); + dump_num(&b, "from", (long)c->site_start_byte); + dump_num(&b, "to", (long)c->site_end_byte); + dump_num(&b, "args", c->arg_count); + } + for (int i = 0; i < r->imports.count; i++) { + dump_str(&b, "\nI", r->imports.items[i].local_name); + dump_str(&b, "path", r->imports.items[i].module_path); + } + dump_str(&b, "\nP", r->error_ranges); + dump_num(&b, "partial", r->parse_incomplete); + dump_num(&b, "unusable", r->parse_unusable); + dump_str(&b, "", "\n"); + return b.buf; +} + +static uint32_t module_end_line(const CBMFileResult *r) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Module") == 0) { + return r->defs.items[i].end_line; + } + } + return 0; +} + +static char *extract_dump(const char *src, CBMLanguage lang, const char *path, + uint32_t *module_end) { + CBMFileResult *r = do_extract(src, lang, path); + if (!r) { + return NULL; + } + char *d = dump_extraction(r); + if (module_end) { + *module_end = module_end_line(r); + } + cbm_free_result(r); + return d; +} + +/* GUARD: for code grammars a trailing newline is insignificant, so a file + * without one must extract byte-identically to the same file with one -- every + * def, range, text field and call byte span. The virtual newline may never + * show up as phantom content or a phantom line. */ +TEST(code_without_final_newline_extracts_identically_issue2078) { + struct { + const char *src; /* no trailing newline */ + CBMLanguage lang; + const char *path; + uint32_t lines; + } cases[] = { + {"#include \n\nstatic int beta(int x) {\n return x + 1;\n}\n\n" + "void alpha(void) { printf(\"a\"); beta(2); }", + CBM_LANG_C, "a.c", 7}, + {"import os\n\n\nclass K:\n def m(self, a):\n return os.path.join(a)\n\n\n" + "def f(x):\n \"\"\"doc\"\"\"\n return K().m(x)", + CBM_LANG_PYTHON, "a.py", 11}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + size_t n = strlen(cases[i].src); + char *terminated = (char *)malloc(n + 2); + ASSERT_NOT_NULL(terminated); + memcpy(terminated, cases[i].src, n); + terminated[n] = '\n'; + terminated[n + 1] = '\0'; + uint32_t bare_module_end = 0; + char *bare = extract_dump(cases[i].src, cases[i].lang, cases[i].path, &bare_module_end); + char *term = extract_dump(terminated, cases[i].lang, cases[i].path, NULL); + free(terminated); + ASSERT_NOT_NULL(bare); + ASSERT_NOT_NULL(term); + bool same = strcmp(bare, term) == 0; + if (!same) { + fprintf(stderr, " %s differs:\n--- without final newline%s--- with final newline%s", + cases[i].path, bare, term); + } + free(bare); + free(term); + if (!same) { + FAIL("extraction must not depend on whether the last line is terminated"); + } + if (bare_module_end != cases[i].lines) { + fprintf(stderr, " %s: module ends on line %u, file has %u\n", cases[i].path, + bare_module_end, cases[i].lines); + FAIL("the virtual newline must not add a line to the module"); + } + } + PASS(); +} + +/* GUARD against phantom content: a heading on an unterminated last line is a + * one-line file. No def may end past line 1, no text may carry the virtual + * newline, and the result must equal the terminated file's. */ +TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078) { + CBMFileResult *r = do_extract("# Top", CBM_LANG_MARKDOWN, "top.md"); + ASSERT_NOT_NULL(r); + bool found = has_def(r, "Top"); + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->end_line > 1 || d->start_line > 1) { + fprintf(stderr, " def %s spans %u-%u\n", d->name ? d->name : "?", d->start_line, + d->end_line); + cbm_free_result(r); + FAIL("no def may reach past the last real line"); + } + if ((d->name && strchr(d->name, '\n')) || (d->signature && strchr(d->signature, '\n'))) { + cbm_free_result(r); + FAIL("no extracted text may contain the virtual newline"); + } + } + cbm_free_result(r); + if (!found) { + /* Before #2078 the unterminated heading was silently dropped: no + * Section, and no parse_partial flag to say so. */ + FAIL("the heading on an unterminated last line must be extracted as a Section"); + } + PASS(); +} + +/* GUARD: the empty file and a file that is exactly "\n" get no virtual + * newline (nothing is unterminated) and stay clean. */ +TEST(empty_and_newline_only_files_unchanged_issue2078) { + const char *srcs[] = {"", "\n"}; + CBMLanguage langs[] = {CBM_LANG_MARKDOWN, CBM_LANG_PYTHON, CBM_LANG_C}; + const char *paths[] = {"e.md", "e.py", "e.c"}; + for (size_t s = 0; s < 2; s++) { + for (size_t l = 0; l < 3; l++) { + CBMFileResult *r = do_extract(srcs[s], langs[l], paths[l]); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->parse_incomplete); + ASSERT_FALSE(r->parse_unusable); + cbm_free_result(r); + } + } + PASS(); +} + SUITE(parse_coverage) { RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete); RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted); @@ -1640,6 +1903,7 @@ SUITE(parse_coverage) { RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610); RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610); RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610); + RUN_TEST(makefile_unterminated_recipe_is_parsed_issue2078); RUN_TEST(c_ifdef_split_range_narrows_to_dropped_branch); RUN_TEST(c_ifdef_split_range_excludes_lines_the_preprocessor_explained); RUN_TEST(c_ifdef_split_range_never_starts_on_a_directive); @@ -1666,4 +1930,9 @@ SUITE(parse_coverage) { RUN_TEST(cs_collection_expression_in_conditional_is_complete_issue1748); RUN_TEST(cs_calls_inside_collection_expression_extracted_issue1748); RUN_TEST(cs_malformed_conditional_remains_partial_issue1748); + RUN_TEST(markdown_unterminated_last_line_is_clean_issue2078); + RUN_TEST(markdown_terminated_last_line_controls_unchanged_issue2078); + RUN_TEST(code_without_final_newline_extracts_identically_issue2078); + RUN_TEST(virtual_newline_adds_no_phantom_line_or_text_issue2078); + RUN_TEST(empty_and_newline_only_files_unchanged_issue2078); } From e066515190a1d1747edb23274c3f1333b80102be Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 28 Sep 2026 19:34:20 +0200 Subject: [PATCH 16/34] fix(routes): mint Laravel routes written without a leading slash Why: Laravel accepts a route URI with or without its leading slash and trims '/' from both ends, so Route::get('users', ...) serves /users and Route::get('', ...) the group root. Most Laravel apps write it that way. Both route passes only minted a Route for a first argument starting with '/', so Firefly III's API registrations (246 distinct routes in routes/api.php) had no Route at all, and neither did laravel.io's PUT/DELETE API routes or 62 of its 67 registrations. Fix (extract_calls.c, PHP only; other languages' path gates are untouched): - A recognised Laravel registration gets its literal URI normalised to the slashed form before the gates: 'users' -> /users, '' -> /, 'users/' -> /users. It is recognised by an HTTP verb (get/post/put/patch/delete) called on the Route facade - directly, or at the root of a registrar chain such as Route::middleware('auth')->get(...) - or on $router. The literal `Route` scope is the same one the #952 callee qualification accepts. Any other call keeps its argument: $request->get('name'), Cache::get('key'), config(), a collection's or session's get(), $this->router->get(), and a test client's $this->get('users'). - The normalised path composes with the enclosing in-file prefix groups the same way as #952. The attribute-array group form Route::group(['prefix' => 'v1/autocomplete', ...], function () {...}), which Firefly III's whole API uses, now contributes its prefix too; without it those routes would mint as /accounts instead of /v1/autocomplete/accounts. - A slashless registration whose action is a string ('index' inside a Route::controller() group, 'UserController@store') is minted without a handler. The string does not say which class owns the method, and resolving it could only guess: on krayin/laravel-crm, 'update' and 'destroy' bound functions in a bundled chart.js, and 'store' bound ActivityController for /login. Proof (release CLI built from this tree vs origin/main 1f9b4db1): - Routes minted by route registration: firefly-iii 30 -> 651, laravel.io 39 -> 81, krayin/laravel-crm 5 -> 132. An independent recomputation of every composed path from the route files matches exactly: firefly routes/api.php 246/246 and routes/web.php 390/390, laravel.io web 64/64 and api 3/3, with 0 missing and 0 extra. - Samples checked against source: GET /v1/autocomplete/accounts and DELETE /v1/accounts/{account} (firefly), PUT /articles/{article} (laravel.io api), PUT /admin/articles/{article}/approve (laravel.io web), and PUT /leads/edit/{id} (krayin). - No false Routes: every new route-registration edge comes from a Route:: registrar in a route file. The pre-existing test-client and JS-client route edges are identical to main. HANDLES stays at 0 on all three corpora, so no handler is guessed. Tests (edge_types_probe): routes_laravel_slashless_issue1146 and routes_laravel_slashless_parallel_issue1146 assert the exact method+path set: facade, facade chain, $router, chain and array prefix groups, '' and a trailing slash. routes_laravel_slashless_no_junk_issue1146 covers the non-route string calls. routes_laravel_slashless_no_guessed_handlers_issue1146 checks string actions against JS and PHP decoys. RED on origin/main 3x, green, RED on revert (the guessed-handler test is also RED with only the handler gate removed). edge_types_probe, php_lsp, pipeline, parallel, route_canon and cross_repo pass (731); make lint-ci passes. Refs #1146 Signed-off-by: Martin Vogel --- internal/cbm/extract_calls.c | 152 +++++++++++++++++++++++++++++++- tests/test_edge_types_probe.c | 159 ++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 2 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index f14efb766..a8b97c639 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -1763,6 +1763,59 @@ static const char *php_chain_prefix_arg(CBMArena *a, TSNode call_node, const cha return NULL; } +/* The value of a plain PHP string literal ('...' or "..." with no + * interpolation or escapes), "" for an empty one; NULL for anything else. */ +static const char *php_plain_string_value(CBMArena *a, TSNode node, const char *source) { + const char *kind = ts_node_type(node); + if (strcmp(kind, "string") != 0 && strcmp(kind, "encapsed_string") != 0) { + return NULL; + } + uint32_t nc = ts_node_named_child_count(node); + for (uint32_t i = 0; i < nc; i++) { + if (strcmp(ts_node_type(ts_node_named_child(node, i)), "string_content") != 0) { + return NULL; + } + } + return nc == 0 ? "" : cbm_node_text(a, ts_node_named_child(node, 0), source); +} + +/* The first argument's value node of a PHP call (unwrapping `argument`). */ +static TSNode php_first_arg_value(TSNode call_node) { + TSNode none = {0}; + TSNode args = ts_node_child_by_field_name(call_node, TS_FIELD("arguments")); + if (ts_node_is_null(args) || ts_node_named_child_count(args) == 0) { + return none; + } + TSNode arg = ts_node_named_child(args, 0); + if (strcmp(ts_node_type(arg), "argument") == 0) { + return ts_node_named_child_count(arg) == SKIP_ONE ? ts_node_named_child(arg, 0) : none; + } + return arg; +} + +/* The literal 'prefix' of an attribute-array group (#1146): + * Route::group(['prefix' => 'v1/autocomplete', 'as' => ...], function () {...}), + * the form Firefly III's whole API is written in. */ +static const char *php_group_array_prefix(CBMArena *a, TSNode group_call, const char *source) { + TSNode attrs = php_first_arg_value(group_call); + if (ts_node_is_null(attrs) || strcmp(ts_node_type(attrs), "array_creation_expression") != 0) { + return NULL; + } + uint32_t nc = ts_node_named_child_count(attrs); + for (uint32_t i = 0; i < nc; i++) { + TSNode el = ts_node_named_child(attrs, i); + if (strcmp(ts_node_type(el), "array_element_initializer") != 0 || + ts_node_named_child_count(el) != PAIR_LEN) { + continue; + } + const char *key = php_plain_string_value(a, ts_node_named_child(el, 0), source); + if (key && strcmp(key, "prefix") == 0) { + return php_plain_string_value(a, ts_node_named_child(el, SKIP_ONE), source); + } + } + return NULL; +} + static const char *php_group_prefix_for_call(CBMArena *a, TSNode node, const char *source) { const char *parts[PHP_PREFIX_PARTS_MAX]; int part_count = 0; @@ -1785,7 +1838,14 @@ static const char *php_group_prefix_for_call(CBMArena *a, TSNode node, const cha strcmp(ts_node_type(p), "scoped_call_expression") == 0)) { TSNode gname = ts_node_child_by_field_name(p, TS_FIELD("name")); char *gtxt = ts_node_is_null(gname) ? NULL : cbm_node_text(a, gname, source); - if (gtxt && strcmp(gtxt, "group") == 0) { + const char *array_prefix = gtxt && strcmp(gtxt, "group") == 0 + ? php_group_array_prefix(a, p, source) + : NULL; + if (array_prefix) { + if (part_count < PHP_PREFIX_PARTS_MAX) { + parts[part_count++] = array_prefix; /* inner-first */ + } + } else if (gtxt && strcmp(gtxt, "group") == 0) { /* Scan the receiver chain for prefix('...'). */ TSNode recv = ts_node_child_by_field_name(p, TS_FIELD("object")); for (int hops = 0; hops < PHP_GROUP_WALK_MAX && !ts_node_is_null(recv); @@ -1843,6 +1903,81 @@ static const char *php_group_prefix_for_call(CBMArena *a, TSNode node, const cha return pos ? cbm_arena_strndup(a, buf, pos) : NULL; } +/* A Laravel route registrar receiver (#1146): the Route facade — also at the + * root of a registrar chain, Route::middleware('auth')->get(...) — or the + * $router instance route files are loaded with (RouteServiceProvider, Lumen). + * The same literal `Route` scope the #952 callee qualification accepts. */ +static bool php_is_laravel_registrar(CBMArena *a, TSNode call_node, const char *source) { + TSNode cur = call_node; + for (int hops = 0; hops < PHP_GROUP_WALK_MAX && !ts_node_is_null(cur); hops++) { + const char *kind = ts_node_type(cur); + if (strcmp(kind, "scoped_call_expression") == 0) { + TSNode scope = ts_node_child_by_field_name(cur, TS_FIELD("scope")); + const char *st = ts_node_is_null(scope) ? NULL : cbm_node_text(a, scope, source); + return st && strcmp(st, "Route") == 0; + } + if (strcmp(kind, "variable_name") == 0) { + const char *vt = cbm_node_text(a, cur, source); + return vt && strcmp(vt, "$router") == 0; + } + if (strcmp(kind, "member_call_expression") != 0) { + return false; + } + cur = ts_node_child_by_field_name(cur, TS_FIELD("object")); + } + return false; +} + +/* Laravel accepts a route URI with or without its leading slash and trims + * '/' from both ends: Route::get('users', ...) serves /users, and '' is the + * group root. The route passes only mint a Route for a path that starts with + * '/', so give a recognised registration's literal URI its slash here, where + * the first argument is still at hand (#1146). Returns NULL — leave the call + * alone — for any other call, any non-literal URI, or one already slashed: + * `$request->get('name')`, `Cache::get('key')` and a collection's get('k') + * keep their argument. */ +static const char *php_laravel_route_uri(CBMArena *a, TSNode call_node, const char *source) { + static const char *const verbs[] = {"get", "post", "put", "patch", "delete", NULL}; + TSNode mname = ts_node_child_by_field_name(call_node, TS_FIELD("name")); + const char *verb = ts_node_is_null(mname) ? NULL : cbm_node_text(a, mname, source); + bool is_verb = false; + for (int i = 0; verb && verbs[i] && !is_verb; i++) { + is_verb = strcmp(verb, verbs[i]) == 0; + } + if (!is_verb || !php_is_laravel_registrar(a, call_node, source)) { + return NULL; + } + TSNode arg0 = php_first_arg_value(call_node); + const char *uri = ts_node_is_null(arg0) ? NULL : php_plain_string_value(a, arg0, source); + if (!uri || uri[0] == '/') { + return NULL; + } + size_t len = strlen(uri); + while (len > 0 && uri[len - SKIP_ONE] == '/') { + len--; + } + return cbm_arena_sprintf(a, "/%.*s", (int)len, uri); +} + +/* Is a Laravel registration's action (its last argument) a string — 'index' + * inside a Route::controller() group, 'UserController@show'? Such an action + * does not say which class owns the method, so resolving it could only guess a + * same-named function anywhere in the repo. */ +static bool php_route_action_is_string(TSNode call_node) { + TSNode args = ts_node_child_by_field_name(call_node, TS_FIELD("arguments")); + uint32_t nc = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + if (nc < PAIR_LEN) { + return false; + } + TSNode action = ts_node_named_child(args, nc - SKIP_ONE); + if (strcmp(ts_node_type(action), "argument") == 0 && + ts_node_named_child_count(action) == SKIP_ONE) { + action = ts_node_named_child(action, 0); + } + const char *kind = ts_node_type(action); + return strcmp(kind, "string") == 0 || strcmp(kind, "encapsed_string") == 0; +} + static bool is_nested_verilog_call_wrapper(CBMLanguage lang, TSNode node) { if (lang != CBM_LANG_VERILOG || strcmp(ts_node_type(node), "subroutine_call") != 0) { return false; @@ -3941,6 +4076,16 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML } if (!ts_node_is_null(args)) { call.first_string_arg = extract_url_or_topic_arg(ctx, args, call.callee_name); + /* #1146: a Laravel registration's slashless URI ('users', '') + * becomes '/users' / '/' so it mints and composes below. */ + const char *laravel_uri = NULL; + if (ctx->language == CBM_LANG_PHP && call.callee_name && + cbm_service_pattern_route_method(call.callee_name) != NULL) { + laravel_uri = php_laravel_route_uri(ctx->arena, node, ctx->source); + if (laravel_uri) { + call.first_string_arg = laravel_uri; + } + } /* #952: routes registered inside Laravel `prefix()->group()` * closures must carry the composed path — the resolve passes * only see the flat CBMCall, so the enclosing chain can only @@ -3959,7 +4104,10 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML : cbm_arena_strndup(ctx->arena, gp, strlen(gp)); } } - if (call.first_string_arg && call.first_string_arg[0] == '/') { + /* A slashless registration's string action would only be + * guessed at (#1146): mint its Route without a handler. */ + if (call.first_string_arg && call.first_string_arg[0] == '/' && + !(laravel_uri && php_route_action_is_string(node))) { call.second_arg_name = extract_handler_arg(ctx, args); } if (ctx->language == CBM_LANG_OBJECTSCRIPT_UDL || diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index 5ed449b8d..30541db2c 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -922,6 +922,161 @@ TEST(routes_laravel_withrouting_parallel_subdir_issue1146) { PASS(); } +/* Assert the exact Route set as "METHOD /path" strings: the method comes from + * the Route QN ("__route__PUT__/articles/{}"), the path from the node name + * (the raw composed path). The count must match too, so a junk Route fails. */ +static int et_route_set_exact(const EtFile *files, int nfiles, const char **want, int parallel) { + EtProj lp; + cbm_store_t *store = + parallel ? et_index_parallel(&lp, files, nfiles) : et_index_files(&lp, files, nfiles); + cbm_node_t *nodes = NULL; + int node_count = 0; + int wanted = 0; + int found[ET_ROUTE_ASSERT_MAX] = {0}; + while (want[wanted] && wanted < ET_ROUTE_ASSERT_MAX) { + wanted++; + } + int ok = store != NULL && + cbm_store_find_nodes_by_label(store, lp.project, "Route", &nodes, &node_count) == + CBM_STORE_OK && + node_count == wanted; + for (int ni = 0; store && ni < node_count; ni++) { + char got[512]; + const char *qn = nodes[ni].qualified_name ? nodes[ni].qualified_name : ""; + const char *m = strncmp(qn, "__route__", 9) == 0 ? qn + 9 : qn; + const char *end = strstr(m, "__"); + snprintf(got, sizeof(got), "%.*s %s", end ? (int)(end - m) : 0, m, + nodes[ni].name ? nodes[ni].name : ""); + int matched = 0; + for (int wi = 0; wi < wanted; wi++) { + if (!found[wi] && strcmp(got, want[wi]) == 0) { + found[wi] = matched = 1; + break; + } + } + if (!matched) { + ok = 0; + fprintf(stderr, " [ET-ROUTESET] unexpected %s\n", got); + } + } + for (int wi = 0; wi < wanted; wi++) { + if (!found[wi]) { + ok = 0; + fprintf(stderr, " [ET-ROUTESET] missing %s\n", want[wi]); + } + } + if (!ok) { + fprintf(stderr, " [ET-ROUTESET] FAIL (%s path) expected=%d actual=%d\n", + parallel ? "parallel" : "sequential", wanted, node_count); + } + cbm_store_free_nodes(nodes, node_count); + et_cleanup(&lp, store); + return ok; +} + +/* #1146: Laravel writes most route URIs without the leading slash — + * Route::get('users', ...) serves /users, Route::put('{article}', ...) inside + * prefix('articles') serves /articles/{article}, and '' is the group root. + * Both route passes only minted a Route for a first argument starting with + * '/', so Firefly III's API (258 registrations) and laravel.io's PUT/DELETE + * API routes had no Route at all. Covers the facade, a facade chain + * (Route::middleware()->get), the $router instance, the chain-form and the + * array-form prefix group (Route::group(['prefix' => ...], fn)), and a + * trailing slash. */ +static const EtFile et_laravel_slashless[] = { + {"routes/api.php", + "group(function () {\n" + " Route::get('', 'ArticlesController@index');\n" + " Route::put('{article}', 'ArticlesController@update');\n" + " Route::delete('{article}', 'ArticlesController@delete');\n" + "});\n" + "Route::group(['prefix' => 'v1/autocomplete', 'as' => 'api.v1.'], static function (): void {\n" + " Route::get('accounts', ['uses' => 'AccountController@accounts', 'as' => 'accounts']);\n" + "});\n" + "Route::middleware('auth')->get('dashboard', 'DashboardController@show');\n" + "$router->get('lumen/users', 'UserController@index');\n"}}; + +static const char *et_laravel_slashless_routes[] = {"GET /users", + "POST /users", + "GET /articles", + "PUT /articles/{article}", + "DELETE /articles/{article}", + "GET /v1/autocomplete/accounts", + "GET /dashboard", + "GET /lumen/users", + NULL}; + +TEST(routes_laravel_slashless_issue1146) { + ASSERT_TRUE(et_route_set_exact(et_laravel_slashless, 1, et_laravel_slashless_routes, 0)); + PASS(); +} + +TEST(routes_laravel_slashless_parallel_issue1146) { + ASSERT_TRUE(et_route_set_exact(et_laravel_slashless, 1, et_laravel_slashless_routes, 1)); + PASS(); +} + +/* The string action of a slashless registration — 'index' inside a + * Route::controller() group, 'UserController@store' — does not say which + * class owns the method, so resolving it could only guess a same-named + * function anywhere in the repo (on krayin/laravel-crm 'update' and 'destroy' + * bound a chart.js function). Such routes are minted without a guessed + * HANDLES edge. Decoys: a JS function and a PHP method per action name. */ +TEST(routes_laravel_slashless_no_guessed_handlers_issue1146) { + static const char *want[] = {"GET /leads", "PUT /leads/edit/{id}", "POST /users", NULL}; + static const EtFile f[] = { + {"routes/web.php", + "prefix('leads')->group(function () {\n" + " Route::get('', 'index');\n" + " Route::put('edit/{id}', 'update');\n" + "});\n" + "Route::post('users', 'UserController@store');\n"}, + {"resources/js/chart.js", + "function index() { return 1; }\nfunction update() { return 2; }\n"}, + {"app/Http/Controllers/ActivityController.php", + "get('name');\n" + " $q = $request->query->get('q');\n" + " $v = Cache::get('users.count');\n" + " $c = config('app.name');\n" + " $k = collect(['a' => 1])->get('a');\n" + " $s = session()->get('key');\n" + " $r = $this->router->get('status');\n" + " $p = $this->post('login', []);\n" + " return $this->get('users');\n" + " }\n}\n"}}; + ASSERT_TRUE(et_route_set_exact(f, 1, want, 0)); + PASS(); +} + /* Rails (Ruby) — ActionDispatch router. The handler MUST be passed as a bare * identifier (not the idiomatic `to: 'list_items'` string, which extract_handler_arg * cannot capture). mapper.get resolves by name to the Mapper#get method whose QN @@ -2138,6 +2293,10 @@ SUITE(edge_types_probe) { RUN_TEST(routes_laravel_routeserviceprovider_control_issue1146); RUN_TEST(routes_laravel_withrouting_nonliteral_prefix_issue1146); RUN_TEST(routes_laravel_withrouting_parallel_subdir_issue1146); + RUN_TEST(routes_laravel_slashless_issue1146); + RUN_TEST(routes_laravel_slashless_parallel_issue1146); + RUN_TEST(routes_laravel_slashless_no_guessed_handlers_issue1146); + RUN_TEST(routes_laravel_slashless_no_junk_issue1146); RUN_TEST(handles_rails_ruby); RUN_TEST(handles_actix_rust); From fc25f3746890ba310abd6d83412c1dcfc85d872d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 17:57:32 +0200 Subject: [PATCH 17/34] fix(extract): narrow the scope of the typedef-pass start index cppcheck 2.20 (the CI linter) reports variableScope for the count taken before the definitions walk, which only the C-family branch uses. The branch now takes the count, walks and drops the shadowed typedefs; the other languages walk as before. Behaviour unchanged: extraction and pipeline suites pass. Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 3d50c95f2..281b38359 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -10498,10 +10498,12 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx) { } // Walk AST for function/class definitions - int first = ctx->result->defs.count; - walk_defs(ctx, ctx->root, spec, 0); if (is_c_declarator_lang(ctx->language)) { + int first = ctx->result->defs.count; + walk_defs(ctx, ctx->root, spec, 0); drop_c_typedefs_shadowed_by_tags(ctx, first); + } else { + walk_defs(ctx, ctx->root, spec, 0); } // Extract module-level variables From d141ee937def3d9ce90e852668a96674f864b332 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 3 Oct 2026 01:31:13 +0200 Subject: [PATCH 18/34] fix(extract): preserve Go router group prefixes Follow literal Group receivers through local bindings and compose their prefixes for both sequential and parallel route extraction. Keep current URL argument handling and reuse the existing Laravel prefix composition. Refs #686 Signed-off-by: Martin Vogel --- internal/cbm/extract_calls.c | 274 +++++++++++++++++++++++++++++----- tests/test_edge_types_probe.c | 81 ++++++++++ 2 files changed, 314 insertions(+), 41 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index f14efb766..89caca047 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -1763,8 +1763,50 @@ static const char *php_chain_prefix_arg(CBMArena *a, TSNode call_node, const cha return NULL; } +/* Compose route-group prefix segments into one path. parts[] is inner-first; + * the result is outer-first with exactly one '/' between segments and a + * leading '/'. Returns NULL when no segment contributes or the result would + * be oversized (the route then stays un-prefixed). Shared by the Laravel + * (#952) and Go router-group (#686) prefix walks. */ +static const char *route_prefix_compose(CBMArena *a, const char *const *parts, int part_count) { + char buf[CBM_SZ_256]; + size_t pos = 0; + for (int i = part_count - 1; i >= 0; i--) { + const char *seg = parts[i]; + while (*seg == '/') { + seg++; + } + size_t sl = strlen(seg); + if (sl == 0) { + continue; + } + if (pos + sl + 2 >= sizeof(buf)) { + return NULL; /* oversized — leave path un-prefixed */ + } + buf[pos++] = '/'; + memcpy(buf + pos, seg, sl); + pos += sl; + while (pos > 1 && buf[pos - 1] == '/') { + pos--; /* strip trailing slash per segment */ + } + } + buf[pos] = '\0'; + return pos ? cbm_arena_strndup(a, buf, pos) : NULL; +} + +/* Prepend a composed group prefix to a route path ("/x" + "/users" -> + * "/x/users"; a bare "/" route becomes the prefix itself). */ +static const char *route_path_with_prefix(CBMArena *a, const char *prefix, const char *path) { + const char *rel = path; + while (*rel == '/') { + rel++; + } + return rel[0] ? cbm_arena_sprintf(a, "%s/%s", prefix, rel) + : cbm_arena_strndup(a, prefix, strlen(prefix)); +} + static const char *php_group_prefix_for_call(CBMArena *a, TSNode node, const char *source) { - const char *parts[PHP_PREFIX_PARTS_MAX]; + const char *parts[PHP_PREFIX_PARTS_MAX] = {NULL}; int part_count = 0; TSNode cur = ts_node_parent(node); for (int depth = 0; depth < PHP_GROUP_WALK_MAX && !ts_node_is_null(cur); depth++) { @@ -1813,34 +1855,200 @@ static const char *php_group_prefix_for_call(CBMArena *a, TSNode node, const cha } cur = ts_node_parent(cur); } - if (part_count == 0) { + return route_prefix_compose(a, parts, part_count); +} + +/* #686: compose Go router-group prefixes (Fiber, Gin, Echo: `g := app.Group("/admin")` + * then `g.Post("/x")`). Starting at the route call's receiver, follow: + * - an identifier to the latest binding before the use (`:=`, `=`, `var`) + * among the preceding statements of the enclosing blocks of the same + * function (a re-bound variable uses the binding in effect at the call), + * - a `recv.Group("", ...)` call to its literal prefix and receiver, + * until the root router (`fiber.New()`, `gin.Default()`, a parameter ...). + * A non-literal group prefix returns NULL: the route stays un-prefixed rather + * than carrying a guessed path. Groups passed across functions are not + * followed (parameters are roots). */ +enum { GO_GROUP_HOPS_MAX = 32 }; + +static const char *go_string_literal_value(CBMArena *a, TSNode n, const char *source) { + const char *k = ts_node_type(n); + if (strcmp(k, "interpreted_string_literal") != 0 && strcmp(k, "raw_string_literal") != 0) { return NULL; } - /* parts[] is inner-first; compose outer-first. Ensure exactly one '/' - * between segments and a leading '/'. */ - char buf[CBM_SZ_256]; - size_t pos = 0; - for (int i = part_count - 1; i >= 0; i--) { - const char *seg = parts[i]; - while (*seg == '/') { - seg++; - } - size_t sl = strlen(seg); - if (sl == 0) { + char *t = cbm_node_text(a, n, source); + size_t len = t ? strlen(t) : 0; + if (len < 2) { + return NULL; + } + t[len - 1] = '\0'; + return t + 1; +} + +/* 1 = `recv.Group("")` (receiver + literal written out), 0 = not a Group + * call, -1 = a Group call whose prefix is not a string literal. */ +static int go_group_call(CBMArena *a, TSNode expr, const char *source, TSNode *recv_out, + const char **prefix_out) { + if (strcmp(ts_node_type(expr), "call_expression") != 0) { + return 0; + } + TSNode fn = ts_node_child_by_field_name(expr, TS_FIELD("function")); + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "selector_expression") != 0) { + return 0; + } + TSNode field = ts_node_child_by_field_name(fn, TS_FIELD("field")); + const char *ft = ts_node_is_null(field) ? NULL : cbm_node_text(a, field, source); + if (!ft || strcmp(ft, "Group") != 0) { + return 0; + } + *recv_out = ts_node_child_by_field_name(fn, TS_FIELD("operand")); + TSNode args = ts_node_child_by_field_name(expr, TS_FIELD("arguments")); + *prefix_out = NULL; + if (!ts_node_is_null(args) && ts_node_named_child_count(args) > 0) { + *prefix_out = go_string_literal_value(a, ts_node_named_child(args, 0), source); + } + return *prefix_out ? 1 : -1; +} + +/* Value bound to `name` by one var_spec (`var a, b = x, y`), or a null node. */ +static TSNode go_var_spec_binding(CBMArena *a, TSNode spec, const char *name, const char *source) { + TSNode none = {0}; + TSNode value = ts_node_child_by_field_name(spec, TS_FIELD("value")); + if (ts_node_is_null(value)) { + return none; + } + uint32_t idx = 0; + uint32_t nc = ts_node_named_child_count(spec); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_named_child(spec, i); + if (strcmp(ts_node_type(c), "identifier") != 0) { continue; } - if (pos + sl + 2 >= sizeof(buf)) { - return NULL; /* oversized — leave path un-prefixed */ + const char *t = cbm_node_text(a, c, source); + if (t && strcmp(t, name) == 0) { + return idx < ts_node_named_child_count(value) ? ts_node_named_child(value, idx) : none; } - buf[pos++] = '/'; - memcpy(buf + pos, seg, sl); - pos += sl; - while (pos > 1 && buf[pos - 1] == '/') { - pos--; /* strip trailing slash per segment */ + idx++; + } + return none; +} + +/* Value bound to `name` by statement `stmt`, or a null node. */ +static TSNode go_stmt_binding(CBMArena *a, TSNode stmt, const char *name, const char *source) { + TSNode none = {0}; + const char *k = ts_node_type(stmt); + if (strcmp(k, "var_declaration") == 0 || strcmp(k, "var_spec_list") == 0) { + for (uint32_t i = ts_node_named_child_count(stmt); i > 0; i--) { + TSNode c = ts_node_named_child(stmt, i - 1); + TSNode v = strcmp(ts_node_type(c), "var_spec") == 0 + ? go_var_spec_binding(a, c, name, source) + : go_stmt_binding(a, c, name, source); + if (!ts_node_is_null(v)) { + return v; + } + } + return none; + } + if (strcmp(k, "short_var_declaration") != 0 && strcmp(k, "assignment_statement") != 0) { + return none; + } + TSNode left = ts_node_child_by_field_name(stmt, TS_FIELD("left")); + TSNode right = ts_node_child_by_field_name(stmt, TS_FIELD("right")); + if (ts_node_is_null(left) || ts_node_is_null(right)) { + return none; + } + uint32_t nl = ts_node_named_child_count(left); + for (uint32_t i = 0; i < nl; i++) { + const char *t = cbm_node_text(a, ts_node_named_child(left, i), source); + if (t && strcmp(t, name) == 0) { + return i < ts_node_named_child_count(right) ? ts_node_named_child(right, i) : none; } } - buf[pos] = '\0'; - return pos ? cbm_arena_strndup(a, buf, pos) : NULL; + return none; +} + +static bool go_is_function_boundary(const char *k) { + return strcmp(k, "function_declaration") == 0 || strcmp(k, "method_declaration") == 0 || + strcmp(k, "func_literal") == 0 || strcmp(k, "source_file") == 0; +} + +/* Latest binding of `name` among the statements preceding `use` in its + * enclosing blocks, innermost block first; a null node when none. */ +static TSNode go_find_binding(CBMArena *a, TSNode use, const char *name, const char *source) { + TSNode child = use; + TSNode cur = ts_node_parent(use); + while (!ts_node_is_null(cur) && !go_is_function_boundary(ts_node_type(cur))) { + const char *k = ts_node_type(cur); + if (strcmp(k, "block") == 0 || strcmp(k, "statement_list") == 0) { + for (TSNode s = ts_node_prev_named_sibling(child); !ts_node_is_null(s); + s = ts_node_prev_named_sibling(s)) { + TSNode v = go_stmt_binding(a, s, name, source); + if (!ts_node_is_null(v)) { + return v; + } + } + } + child = cur; + cur = ts_node_parent(cur); + } + TSNode none = {0}; + return none; +} + +static const char *go_group_prefix_for_call(CBMArena *a, TSNode call, const char *source) { + TSNode fn = ts_node_child_by_field_name(call, TS_FIELD("function")); + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "selector_expression") != 0) { + return NULL; + } + const char *parts[GO_GROUP_HOPS_MAX] = {NULL}; + int part_count = 0; + TSNode recv = ts_node_child_by_field_name(fn, TS_FIELD("operand")); + for (int hop = 0; !ts_node_is_null(recv); hop++) { + if (hop >= GO_GROUP_HOPS_MAX) { + return NULL; /* chain too deep to trust: never a partial prefix */ + } + const char *k = ts_node_type(recv); + if (strcmp(k, "parenthesized_expression") == 0 && ts_node_named_child_count(recv) > 0) { + recv = ts_node_named_child(recv, 0); + continue; + } + if (strcmp(k, "identifier") == 0) { + const char *name = cbm_node_text(a, recv, source); + recv = name ? go_find_binding(a, recv, name, source) : (TSNode){0}; + continue; /* unbound (parameter, package var): root reached */ + } + TSNode inner = {0}; + const char *prefix = NULL; + int g = go_group_call(a, recv, source, &inner, &prefix); + if (g < 0) { + return NULL; /* dynamic prefix: do not guess */ + } + if (g == 0) { + break; /* root router constructor or other expression */ + } + parts[part_count++] = prefix; /* inner-first */ + recv = inner; + } + return route_prefix_compose(a, parts, part_count); +} + +/* Routes registered inside router groups must carry the composed path — the + * resolve passes only see the flat CBMCall, so the enclosing group chain can + * only be read here where the AST still exists. Laravel `prefix()->group()` + * closures (#952) and Go `x.Group("/p")` receivers (#686). Returns the route + * path to store (unchanged when no group prefix applies). */ +static const char *route_path_with_group_prefix(CBMExtractCtx *ctx, TSNode node, + const char *callee_name, const char *path) { + if (!path || path[0] != '/' || !callee_name || + cbm_service_pattern_route_method(callee_name) == NULL) { + return path; + } + const char *gp = NULL; + if (ctx->language == CBM_LANG_PHP) { + gp = php_group_prefix_for_call(ctx->arena, node, ctx->source); + } else if (ctx->language == CBM_LANG_GO) { + gp = go_group_prefix_for_call(ctx->arena, node, ctx->source); + } + return (gp && gp[0]) ? route_path_with_prefix(ctx->arena, gp, path) : path; } static bool is_nested_verilog_call_wrapper(CBMLanguage lang, TSNode node) { @@ -3940,25 +4148,9 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML args = swift_call_args(node); } if (!ts_node_is_null(args)) { - call.first_string_arg = extract_url_or_topic_arg(ctx, args, call.callee_name); - /* #952: routes registered inside Laravel `prefix()->group()` - * closures must carry the composed path — the resolve passes - * only see the flat CBMCall, so the enclosing chain can only - * be read here where the AST still exists. */ - if (ctx->language == CBM_LANG_PHP && call.first_string_arg && - call.first_string_arg[0] == '/' && call.callee_name && - cbm_service_pattern_route_method(call.callee_name) != NULL) { - const char *gp = php_group_prefix_for_call(ctx->arena, node, ctx->source); - if (gp && gp[0]) { - const char *rel = call.first_string_arg; - while (*rel == '/') { - rel++; - } - call.first_string_arg = - rel[0] ? cbm_arena_sprintf(ctx->arena, "%s/%s", gp, rel) - : cbm_arena_strndup(ctx->arena, gp, strlen(gp)); - } - } + call.first_string_arg = route_path_with_group_prefix( + ctx, node, call.callee_name, + extract_url_or_topic_arg(ctx, args, call.callee_name)); if (call.first_string_arg && call.first_string_arg[0] == '/') { call.second_arg_name = extract_handler_arg(ctx, args); } diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index 5ed449b8d..26a9986ae 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -546,6 +546,84 @@ TEST(handles_gin_go) { PASS(); } +/* #686: Go router groups. Fiber (and Gin/Echo, same syntax) build the external + * path from `grp := app.Group("/admin")` plus `grp.Post("/x")`, so the full + * path is never one literal at the registration site. The Route must carry + * the composed path, including nested groups (`v1 := api.Group("/v1")`) and + * groups built inline (`app.Group("/x").Get(...)`). Controls: a route on the + * root app keeps its own path, a variable later re-bound to a different group + * uses the binding in effect at the call, and a non-literal group prefix + * leaves the route unprefixed (never guessed). Exact Route set on BOTH the + * sequential and the parallel pipeline. */ +static const EtFile et_fiber_groups_issue686[] = { + {"main.go", + "package main\n\n" + "import \"github.com/gofiber/fiber/v2\"\n\n" + "func health(c *fiber.Ctx) error { return nil }\n" + "func getUser(c *fiber.Ctx) error { return nil }\n" + "func updateCustomer(c *fiber.Ctx) error { return nil }\n" + "func listOrders(c *fiber.Ctx) error { return nil }\n" + "func ping(c *fiber.Ctx) error { return nil }\n" + "func report(c *fiber.Ctx) error { return nil }\n" + "func dyn(c *fiber.Ctx) error { return nil }\n\n" + "func main() {\n" + " app := fiber.New()\n" + " app.Get(\"/health\", health)\n" + " api := app.Group(\"/api\")\n" + " v1 := api.Group(\"/v1\", authMiddleware)\n" + " v1.Get(\"/users/:id\", getUser)\n" + " admin := app.Group(\"/admin\")\n" + " admin.Post(\"/customers/:id\", updateCustomer)\n" + " grp := api.Group(\"/shop\")\n" + " grp.Get(\"/orders\", listOrders)\n" + " grp = app.Group(\"/internal\")\n" + " grp.Get(\"/ping\", ping)\n" + " app.Group(\"/reports\").Get(\"/daily\", report)\n" + " prefix := \"/p\"\n" + " dg := app.Group(prefix)\n" + " dg.Get(\"/dyn\", dyn)\n" + " app.Listen(\":3000\")\n" + "}\n\n" + "func authMiddleware(c *fiber.Ctx) error { return c.Next() }\n"}}; + +static const char *et_fiber_groups_issue686_routes[] = { + "/health", "/api/v1/users/:id", "/admin/customers/:id", "/api/shop/orders", + "/internal/ping", "/reports/daily", "/dyn", NULL}; + +TEST(handles_fiber_group_prefix_sequential_issue686) { + ASSERT_TRUE(et_routes_exact_mode(et_fiber_groups_issue686, 1, et_fiber_groups_issue686_routes, + false, NULL)); + PASS(); +} + +TEST(handles_fiber_group_prefix_parallel_issue686) { + ASSERT_TRUE(et_routes_exact_mode(et_fiber_groups_issue686, 1, et_fiber_groups_issue686_routes, + true, NULL)); + PASS(); +} + +/* #686 control for the same mechanism on Gin: `v1 := r.Group("/v1")` with the + * idiomatic brace block. */ +TEST(handles_gin_group_prefix_issue686) { + static const EtFile f[] = { + {"main.go", + "package main\n\n" + "import \"github.com/gin-gonic/gin\"\n\n" + "func listOrders(c *gin.Context) {}\n" + "func createOrder(c *gin.Context) {}\n\n" + "func main() {\n" + " r := gin.Default()\n" + " v1 := r.Group(\"/v1\")\n" + " {\n" + " v1.GET(\"/orders\", listOrders)\n" + " v1.POST(\"/orders\", createOrder)\n" + " }\n" + "}\n"}}; + static const char *routes[] = {"/v1/orders", "/v1/orders", NULL}; /* GET + POST */ + ASSERT_TRUE(et_routes_exact(f, 1, routes)); + PASS(); +} + /* Spring (Java) — class-level @RequestMapping must prefix method mappings. * Reproduce-first: a HANDLES count alone can pass with partial routes * ("/orders"), but callers/search_graph need the actual endpoint names @@ -2121,6 +2199,9 @@ SUITE(edge_types_probe) { RUN_TEST(handles_express_ts); RUN_TEST(handles_fastify_js); RUN_TEST(handles_gin_go); + RUN_TEST(handles_fiber_group_prefix_sequential_issue686); + RUN_TEST(handles_fiber_group_prefix_parallel_issue686); + RUN_TEST(handles_gin_group_prefix_issue686); RUN_TEST(handles_spring_java); RUN_TEST(handles_spring_java_path_attribute_fourth); RUN_TEST(handles_spring_kotlin); From 1522c160510912b30d114e6c435ec17a52b8cdfc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 18:11:07 +0200 Subject: [PATCH 19/34] test(httpd): expect the resolved allowed root in its platform form ui_server_index_start_stores_resolved_root built its expected root as the resolved parent plus "/allowed". On Windows the server stores the resolved form of the directory itself, which uses backslashes throughout, so the comparison saw a mixed-separator string and failed on the windows-latest CLANG64 job. The test now resolves the allowed directory with cbm_canonical_path, as the server does, and compares the 202 reply against the JSON-escaped form, as the status-listing check already did; a backslash appears escaped in both bodies. Production is unchanged; on POSIX the expected strings are identical to before. Signed-off-by: Martin Vogel --- tests/test_httpd.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_httpd.c b/tests/test_httpd.c index d020dc7da..4408f4ef5 100644 --- a/tests/test_httpd.c +++ b/tests/test_httpd.c @@ -1188,10 +1188,13 @@ TEST(ui_server_index_start_stores_resolved_root) { char inner[512]; snprintf(inner, sizeof(inner), "%s/allowed/inner", base); ASSERT_EQ(th_mkdir_p(inner), 0); - char canonical_base[4096]; - ASSERT_TRUE(cbm_canonical_path(base, canonical_base, sizeof(canonical_base))); - char allowed_root[4096 + 16]; - snprintf(allowed_root, sizeof(allowed_root), "%s/allowed", canonical_base); + /* The resolved form of the allowed directory itself, as the server + * computes it. A "/allowed" suffix on a resolved parent is not that form + * on Windows, where the resolved path uses backslashes throughout. */ + char allowed_dir[512]; + snprintf(allowed_dir, sizeof(allowed_dir), "%s/allowed", base); + char allowed_root[4096]; + ASSERT_TRUE(cbm_canonical_path(allowed_dir, allowed_root, sizeof(allowed_root))); /* The request names the allowed root through a ".." segment, so its * spelling is not the resolved form (on macOS /tmp is itself a link to * /private/tmp, so the prefix differs as well). */ @@ -1212,12 +1215,13 @@ TEST(ui_server_index_start_stores_resolved_root) { * spelling. */ ASSERT_STR_EQ(probe.executor.root_path, allowed_root); ASSERT_STR_EQ(probe.executor.project_name, "resolved"); - /* The 202 reply and the status listing report that same stored root. */ - char expected[8192 + 64]; - snprintf(expected, sizeof(expected), "\"path\":\"%s\"}", allowed_root); - ASSERT_NOT_NULL(strstr(probe.response, expected)); + /* The 202 reply and the status listing report that same stored root, + * JSON-escaped (a Windows path carries backslashes). */ char escaped[8192]; cbm_json_escape(escaped, (int)sizeof(escaped), allowed_root); + char expected[8192 + 64]; + snprintf(expected, sizeof(expected), "\"path\":\"%s\"}", escaped); + ASSERT_NOT_NULL(strstr(probe.response, expected)); snprintf(expected, sizeof(expected), "\"path\":\"%s\",\"error\"", escaped); ASSERT_NOT_NULL(strstr(probe.index_status, expected)); PASS(); From 14ed8ce38a537a3ce14cb8957d6418ef9a460bae Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 25 Sep 2026 18:37:16 +0200 Subject: [PATCH 20/34] fix(coverage): one line counter for a source buffer (#1967) Three places in internal/cbm/cbm.c counted the lines of the same source buffer with two different conventions. cbm_count_lines (parse_unusable 80% threshold) ignored a final newline, while the inline counter that sizes the preprocessed-line map (orig_lines) and cbm_line_offsets (the recovery-gap walker) counted every newline, so on any file ending in a newline -- nearly every file -- they disagreed by one. Nothing produced a wrong answer: every consumer of the larger count either clamps to it or sees the phantom last line as blank. The risk was the next reader picking up one convention without knowing the other existed. Add cbm_source_line_count() as the single helper, with the convention documented beside it: a '\n' terminates a line and opens no new one; an empty buffer counts as 1 line. All three call sites use it; the old static cbm_count_lines is gone. Test: parse_coverage::source_line_count_one_convention pins the convention (trailing newline, no final newline, CRLF, empty buffer, blank last line, src_len bound). Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 48 ++++++++++++++++++------------------- internal/cbm/cbm.h | 5 ++++ tests/test_parse_coverage.c | 18 ++++++++++++++ 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 4d01d0c69..aae9d64ba 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1433,6 +1433,26 @@ static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, } } +/* The ONE line counter for a source buffer (#1967). Every caller that asks + * "how many lines does this file have" uses it, so the coverage report never + * holds two answers for the same buffer. + * + * Convention: a '\n' TERMINATES a line; it does not open a new one. So the + * count is the number of '\n' that have at least one byte after them, plus + * one. "a\nb" and "a\nb\n" both have 2 lines (the trailing newline adds no + * phantom empty line, matching what an editor shows); "a\nb" without a final + * newline still counts its last line. An empty buffer counts as 1 line, because + * tree-sitter still reports row 0 for it and 1-based line maps index line 1. */ +uint32_t cbm_source_line_count(const char *src, int src_len) { + uint32_t n = 1; + for (int i = 0; i + 1 < src_len; i++) { + if (src[i] == '\n') { + n++; + } + } + return n; +} + /* Recovery subtraction (#963): tree-sitter error recovery plus the * ERROR-descending def walker often still extract constructs INSIDE a failed * region (verified: a function in an #ifdef-split ERROR region and even a @@ -1450,12 +1470,7 @@ static void cbm_mark_pp_error_rows(TSNode n, uint8_t *rows, uint32_t row_count, * Now the uncovered gaps are reported instead, and a gap holding only blank, * comment or preprocessor lines is not a miss at all. */ static uint32_t *cbm_line_offsets(const char *src, int src_len, uint32_t *out_lines) { - uint32_t lines = 1; - for (int i = 0; i < src_len; i++) { - if (src[i] == '\n') { - lines++; - } - } + uint32_t lines = cbm_source_line_count(src, src_len); uint32_t *offs = (uint32_t *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(lines + 1) * sizeof(uint32_t)); if (!offs) { @@ -2060,18 +2075,6 @@ static void cbm_refine_regions_with_pp_lines(cbm_error_regions_t *regs, const ui * this repo covers 25.5% of its file, and the next widest 3.9%. */ #define CBM_UNUSABLE_PCT 80 -/* Number of 1-based lines in `src`. A file that does not end with a newline - * still has a last line, so the count is separators plus one. */ -static uint32_t cbm_count_lines(const char *src, int src_len) { - uint32_t n = 1; - for (int i = 0; i < src_len; i++) { - if (src[i] == '\n' && i + 1 < src_len) { - n++; - } - } - return n; -} - /* Serialize collected regions as "start-end,start-end,...", with a trailing * ",+" when the cap threw N ranges away. * @@ -2745,12 +2748,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * total loss (root is ERROR), because then it vouches for * nothing and there is no refinement to make. */ if (strcmp(ts_node_type(pp_root), "ERROR") != 0) { - uint32_t orig_lines = 1; - for (int ci = 0; ci < source_len; ci++) { - if (source[ci] == '\n') { - orig_lines++; - } - } + uint32_t orig_lines = cbm_source_line_count(source, source_len); uint8_t *map = (uint8_t *)cbm_arena_alloc(a, (size_t)orig_lines + 2); int exp_lines = preprocessed->expanded_line_count; uint8_t *bad_rows = @@ -2946,7 +2944,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C * so the report can say "read the source" instead. See * parse_unusable in cbm.h for which files land here and why. */ if (regs.count == 1 && regs.dropped == 0) { - uint32_t total = cbm_count_lines(source, source_len); + uint32_t total = cbm_source_line_count(source, source_len); uint32_t span = regs.ends[0] - regs.starts[0] + 1; if (total > 0 && span * 100 >= total * CBM_UNUSABLE_PCT) { result->parse_unusable = true; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index dc89a6b2d..7f1aad43e 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -920,6 +920,11 @@ uint64_t cbm_usage_field_lookup_test_work(void); uint64_t cbm_usage_slow_parent_fallback_test_count(void); #endif +// Number of 1-based lines in a source buffer. The single line-count convention +// for coverage reporting (#1967): a trailing '\n' ends the last line and opens +// no new one; an empty buffer counts as 1 line. See the definition in cbm.c. +uint32_t cbm_source_line_count(const char *src, int src_len); + // Toggle C/C++ preprocessor Macro-node extraction (#375). The pipeline enables // it only for full/advanced index modes (it dominates extraction on macro-dense // codebases). Default ON. Set before extraction; read-only during. diff --git a/tests/test_parse_coverage.c b/tests/test_parse_coverage.c index 09caeaa82..5dbc38e4a 100644 --- a/tests/test_parse_coverage.c +++ b/tests/test_parse_coverage.c @@ -1615,7 +1615,25 @@ TEST(cs_malformed_conditional_remains_partial_issue1748) { PASS(); } +/* #1967: every line counter over one source buffer answers the same + * question the same way. A trailing newline ends the last line; it does not + * open a new one. */ +TEST(source_line_count_one_convention) { + ASSERT_EQ(cbm_source_line_count("", 0), 1u); + ASSERT_EQ(cbm_source_line_count("a", 1), 1u); + ASSERT_EQ(cbm_source_line_count("a\n", 2), 1u); + ASSERT_EQ(cbm_source_line_count("\n", 1), 1u); + ASSERT_EQ(cbm_source_line_count("a\nb", 3), 2u); + ASSERT_EQ(cbm_source_line_count("a\nb\n", 4), 2u); + ASSERT_EQ(cbm_source_line_count("a\n\n", 3), 2u); + ASSERT_EQ(cbm_source_line_count("a\r\nb\r\n", 6), 2u); + /* src_len bounds the count, not a NUL terminator. */ + ASSERT_EQ(cbm_source_line_count("a\nb\nc", 2), 1u); + PASS(); +} + SUITE(parse_coverage) { + RUN_TEST(source_line_count_one_convention); RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete); RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted); RUN_TEST(c_error_range_points_at_failed_region); From c5a3c6eae944bb863bd9310faf6a995f4a7365d9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 20 Sep 2026 18:16:02 +0200 Subject: [PATCH 21/34] fix(routes): register FastAPI and Starlette WebSocket decorators Distilled from #1245 by Andrew Hundt (a slice of 1ac3c136). The route-registration suffix table knew .add_api_websocket_route but not the decorator forms FastAPI and Starlette actually use: @router.websocket("/ws") @app.websocket_route("/events") so a WebSocket endpoint produced no Route node on either resolver path and was invisible to the graph. Two entries join the table, both recorded as ANY like the existing WebSocket entry -- a socket upgrade is not an HTTP verb. Both are needed: matching is a pure suffix compare and .route does not match ...websocket_route, whose tail is _route. Upstream also changed pass_calls.c, pass_parallel.c and pipeline_internal.h. None of that is taken: main already carries the #952 sequential unresolved-callee route fallback its July base lacked, so the table entries are the only missing piece. RED before the fix, each form recorded on its own (the harness stops at the first failing assert, so the two were swapped and re-run): pipeline_python_websocket_decorators_register_routes FAIL tests/test_pipeline.c:6163: count_nodes_named(s, project, "/ws") (0) not >= 1 (1) FAIL tests/test_pipeline.c:6163: count_nodes_named(s, project, "/events") (0) not >= 1 (1) The @app.get("/health") control above those lines passed before and after. GREEN after: pipeline 282 passed; pipeline parallel extraction edge_types_probe route_canon infrascan 779 passed, 0 failed. The test drives the sequential resolver (one file). The parallel resolver reads the same table through cbm_service_pattern_route_method and its suite is green; a CBM_WORKERS=4 twin of this test is a small follow-up. Co-authored-by: Andrew Hundt Signed-off-by: Martin Vogel --- internal/cbm/service_patterns.c | 5 ++++ tests/test_pipeline.c | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index 6da74d478..2626b52ca 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -577,6 +577,11 @@ static const method_suffix_t route_reg_suffixes[] = { {".add_route", "ANY"}, {".add_api_route", "ANY"}, {".add_api_websocket_route", "ANY"}, + /* FastAPI/Starlette WebSocket decorators (`@router.websocket('/ws')`, + * `@app.websocket_route('/events')`): not an HTTP verb, so ANY — mirrors + * .add_api_websocket_route above (#1245). */ + {".websocket", "ANY"}, + {".websocket_route", "ANY"}, {NULL, NULL}, }; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 44bb49e9c..889bffbe1 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -7487,6 +7487,56 @@ TEST(pipeline_capitalized_axios_import_is_http_client_sequential) { return capitalized_axios_case(0); } +/* FastAPI/Starlette WebSocket route registration (distilled from #1245). The + * route-registration suffix table knew `.add_api_websocket_route` but not the + * decorator forms `.websocket` / `.websocket_route`, so a WebSocket endpoint + * registered via `@router.websocket('/ws')` produced no Route node at all — + * the endpoint was invisible to the graph. `@app.get('/health')` is the + * positive control (already a route suffix). Fewer than 50 files exercises + * pass_calls.c; the same suffix table drives the parallel resolver. */ +TEST(pipeline_python_websocket_decorators_register_routes) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_ws_routes_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "app.py", + "from fastapi import APIRouter, FastAPI, WebSocket\n\n" + "app = FastAPI()\n" + "router = APIRouter()\n\n" + "@app.get('/health')\n" + "def health():\n" + " return {'ok': True}\n\n" + "@router.websocket('/ws')\n" + "async def ws_endpoint(websocket: WebSocket):\n" + " await websocket.accept()\n\n" + "@app.websocket_route('/events')\n" + "async def events_endpoint(websocket: WebSocket):\n" + " await websocket.accept()\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/py_ws.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* Positive control: the HTTP decorator mints its Route node. */ + ASSERT_GTE(count_nodes_named(s, project, "/health"), 1); + /* The two WebSocket decorator forms must mint Route nodes too. */ + ASSERT_GTE(count_nodes_named(s, project, "/ws"), 1); + ASSERT_GTE(count_nodes_named(s, project, "/events"), 1); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Python bare-call local-binding suppression, sequential path. The bare-call * counterpart of the receiver guard above: `run` is a PARAMETER, so `run()` * cannot be the module-level `run` and must not bind SatoriLive.run. @@ -17568,6 +17618,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges); RUN_TEST(pipeline_capitalized_axios_import_is_http_client); RUN_TEST(pipeline_capitalized_axios_import_is_http_client_sequential); + RUN_TEST(pipeline_python_websocket_decorators_register_routes); RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges); RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge); RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); From c442028010bee9b4ae4e24905fad7fdcd8966b34 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 21:36:08 +0200 Subject: [PATCH 22/34] fix(workspace): classify manifest entries in their resolved form A project's .cbmpathwhitelist lists extra roots it would like indexed. When such a manifest was approved, and again each time it was consulted, every entry was classified exactly as written in the file, while the directory that would actually be indexed is the one the entry resolves to. An entry that reached the home directory, a credential directory or the volume root through a link or through `..` segments was judged by its spelling and passed as an ordinary path; only the containment check resolved it. An entry naming a directory that did not exist was approved as well. Both cbm_workspace_manifest_approve and cbm_workspace_manifest_allows now resolve each entry with cbm_canonical_path, the resolver every other caller of the policy applies, and classify that form. Approval refuses an entry that is relative, does not exist or is not a directory, with a message that names the entry and, when the resolved directory differs from what was written, shows both. Approval stays bound to the manifest text; where an entry leads is read again when it is used, so a link that is retargeted after approval is judged by its new target. At use time an entry that no longer resolves matches nothing, and the other entries of the manifest are consulted as before. An ordinary, existing, absolute entry keeps its result. A manifest that names a directory which does not exist yet is now refused at approval and has to be approved once the directory is there. Signed-off-by: Martin Vogel --- src/foundation/workspace.c | 69 ++++++- src/foundation/workspace.h | 15 +- tests/test_workspace.c | 384 +++++++++++++++++++++++++++++++++++-- 3 files changed, 441 insertions(+), 27 deletions(-) diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index 40e009a03..eab52d48f 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -776,6 +776,43 @@ bool cbm_workspace_manifest_is_approved(const char *cache_dir, const char *proje return found; } +/* Resolve a manifest entry to the directory it names — the form the policy is + * defined over. The file holds the entry as a person wrote it, and a spelling + * can reach a directory through a link or through `..` segments; the directory + * that would be indexed is the resolved one, so that is the one classified, at + * approval and again at use. Only an absolute entry is accepted: the manifest + * is read by a long-lived process whose working directory is unrelated to the + * project, so a relative spelling has no base it could honestly be resolved + * against. Returns NULL on success, otherwise the reason the entry is unusable. + * out must hold WS_LINE_MAX bytes. */ +static const char *ws_manifest_entry_resolve(const char *entry, char *out, size_t out_sz) { + if (ws_volume_prefix_len(entry) == 0) { + return "must be an absolute path"; + } + if (!cbm_canonical_path(entry, out, out_sz)) { + return "must name an existing directory"; + } + if (!cbm_is_dir(out)) { + return "is not a directory"; + } + return NULL; +} + +/* "requested path : ", with the resolved form shown when it + * differs from what was written: the person typed one and the policy judged + * the other. */ +static void ws_manifest_entry_refusal(char *err, size_t err_sz, const char *entry, + const char *resolved, const char *reason) { + if (!err || err_sz == 0) { + return; + } + if (resolved && strcmp(resolved, entry) != 0) { + snprintf(err, err_sz, "requested path %s (resolves to %s): %s", entry, resolved, reason); + } else { + snprintf(err, err_sz, "requested path %s: %s", entry, reason); + } +} + bool cbm_workspace_manifest_approve(const char *cache_dir, const char *home_dir, const char *project_root, char *err, size_t err_sz) { if (err && err_sz) { @@ -800,14 +837,19 @@ bool cbm_workspace_manifest_approve(const char *cache_dir, const char *home_dir, } /* Approving a manifest must not become a way around the breadth policy: every - * requested entry has to stand on its own as an indexing root. */ + * requested entry has to stand on its own as an indexing root, judged by the + * directory it resolves to. */ for (int i = 0; i < m.count; i++) { - cbm_ws_verdict_t v = cbm_workspace_classify_root(m.entries[i], home_dir, cache_dir); + char resolved[WS_LINE_MAX]; + const char *unusable = ws_manifest_entry_resolve(m.entries[i], resolved, sizeof(resolved)); + if (unusable) { + ws_manifest_entry_refusal(err, err_sz, m.entries[i], NULL, unusable); + return false; + } + cbm_ws_verdict_t v = cbm_workspace_classify_root(resolved, home_dir, cache_dir); if (v != CBM_WS_ALLOW) { - if (err) { - snprintf(err, err_sz, "requested path %s: %s", m.entries[i], - cbm_workspace_verdict_reason(v)); - } + ws_manifest_entry_refusal(err, err_sz, m.entries[i], resolved, + cbm_workspace_verdict_reason(v)); return false; } } @@ -850,12 +892,19 @@ bool cbm_workspace_manifest_allows(const char *cache_dir, const char *home_dir, return false; } for (int i = 0; i < m.count; i++) { - /* Re-classify at use time as well as at approval time: the credential list - * may have grown since, and a stored approval must not outrank it. */ - if (cbm_workspace_classify_root(m.entries[i], home_dir, cache_dir) != CBM_WS_ALLOW) { + /* Resolve and re-classify at use time as well as at approval time: the + * credential list may have grown since, and a link may lead somewhere + * else than it did when the person approved it. A stored approval must + * not outrank either. An entry that no longer resolves matches nothing, + * and the other entries of the manifest are consulted as before. */ + char resolved[WS_LINE_MAX]; + if (ws_manifest_entry_resolve(m.entries[i], resolved, sizeof(resolved)) != NULL) { continue; } - if (cbm_path_within_root(m.entries[i], candidate)) { + if (cbm_workspace_classify_root(resolved, home_dir, cache_dir) != CBM_WS_ALLOW) { + continue; + } + if (cbm_path_within_root(resolved, candidate)) { return true; } } diff --git a/src/foundation/workspace.h b/src/foundation/workspace.h index 276079155..c4674696d 100644 --- a/src/foundation/workspace.h +++ b/src/foundation/workspace.h @@ -145,12 +145,19 @@ bool cbm_workspace_manifest_read(const char *project_root, cbm_ws_manifest_t *ou bool cbm_workspace_manifest_is_approved(const char *cache_dir, const char *project_root, const cbm_ws_manifest_t *manifest); -/* Record approval for the manifest currently on disk. Refuses when a requested - * entry would not be allowable as a root on its own — approving a manifest must - * not become a way around the breadth policy. */ +/* Record approval for the manifest currently on disk. Each entry is judged in + * its resolved form (links followed, `..` collapsed), since that is the + * directory that would be indexed; an entry that is relative, does not exist + * or is not a directory is refused, as is one that would not be allowable as a + * root on its own — approving a manifest must not become a way around the + * breadth policy. err names the entry as written and, when it differs, the + * directory it resolves to. */ bool cbm_workspace_manifest_approve(const char *cache_dir, const char *home_dir, const char *project_root, char *err, size_t err_sz); -/* True when candidate is at or below an APPROVED manifest entry of project_root. */ +/* True when candidate is at or below an APPROVED manifest entry of project_root. + * Entries are resolved and classified again here: approval binds to the + * manifest text, and where an entry leads is read at the time it is used. An + * entry that no longer resolves matches nothing. */ bool cbm_workspace_manifest_allows(const char *cache_dir, const char *home_dir, const char *project_root, const char *candidate); diff --git a/tests/test_workspace.c b/tests/test_workspace.c index afe6726e2..7f163db29 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -334,13 +334,29 @@ TEST(ws_manifest_rejects_control_characters) { /* THE property the design rests on: a manifest grants nothing until a person * approves it, and editing it lapses that approval rather than inheriting it. */ TEST(ws_manifest_approval_is_keyed_to_content) { - char *base = th_mktempdir("cbm_ws_m3"); - char *cache = th_mktempdir("cbm_ws_m3c"); - ASSERT(base != NULL); - ASSERT(cache != NULL); + /* th_mktempdir returns one static buffer: copy each result at once. */ + char base[256]; + char cache[256]; + char *created = th_mktempdir("cbm_ws_m3"); + ASSERT(created != NULL); + snprintf(base, sizeof(base), "%s", created); + created = th_mktempdir("cbm_ws_m3c"); + ASSERT(created != NULL); + snprintf(cache, sizeof(cache), "%s", created); + + /* Entries must name existing directories. */ + char sdk[1024]; + char protos[1024]; + snprintf(sdk, sizeof(sdk), "%s/sdk", base); + snprintf(protos, sizeof(protos), "%s/protos", base); + ASSERT_EQ(cbm_mkdir(sdk), 0); + ASSERT_EQ(cbm_mkdir(protos), 0); + char path[1024]; + char body[4096]; snprintf(path, sizeof(path), "%s/%s", base, CBM_WS_MANIFEST_NAME); - ws_write(path, "/opt/sdk\n"); + snprintf(body, sizeof(body), "%s\n", sdk); + ws_write(path, body); cbm_ws_manifest_t before; ASSERT_TRUE(cbm_workspace_manifest_read(base, &before)); @@ -352,7 +368,8 @@ TEST(ws_manifest_approval_is_keyed_to_content) { ASSERT_TRUE(cbm_workspace_manifest_is_approved(cache, base, &before)); /* Widen the requests, as a `git pull` would. Approval must lapse. */ - ws_write(path, "/opt/sdk\n/srv/protos\n"); + snprintf(body, sizeof(body), "%s\n%s\n", sdk, protos); + ws_write(path, body); cbm_ws_manifest_t after; ASSERT_TRUE(cbm_workspace_manifest_read(base, &after)); ASSERT_TRUE(strcmp(before.digest, after.digest) != 0); @@ -363,18 +380,35 @@ TEST(ws_manifest_approval_is_keyed_to_content) { PASS(); } -/* Approving a manifest must not become a route around the breadth policy. */ +/* Approving a manifest must not become a route around the breadth policy. The + * entry has to exist, so the overbroad directory is "/etc" on POSIX and, on + * Windows, where one component below the drive is already an ordinary + * workspace, the drive root of the temporary directory. */ TEST(ws_manifest_approval_refuses_overbroad_requests) { - char *base = th_mktempdir("cbm_ws_m4"); - char *cache = th_mktempdir("cbm_ws_m4c"); - ASSERT(base != NULL); - ASSERT(cache != NULL); + char base[256]; + char cache[256]; + char *created = th_mktempdir("cbm_ws_m4"); + ASSERT(created != NULL); + snprintf(base, sizeof(base), "%s", created); + created = th_mktempdir("cbm_ws_m4c"); + ASSERT(created != NULL); + snprintf(cache, sizeof(cache), "%s", created); + char path[1024]; + char body[1024]; + const char *refusal = NULL; snprintf(path, sizeof(path), "%s/%s", base, CBM_WS_MANIFEST_NAME); - ws_write(path, "/etc\n"); +#ifdef _WIN32 + snprintf(body, sizeof(body), "%.3s\n", base); + refusal = cbm_workspace_verdict_reason(CBM_WS_DENY_ABSOLUTE); +#else + snprintf(body, sizeof(body), "/etc\n"); + refusal = cbm_workspace_verdict_reason(CBM_WS_DENY_TOO_SHALLOW); +#endif + ws_write(path, body); char err[1024]; ASSERT_FALSE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); - ASSERT_TRUE(strstr(err, "too broad") != NULL); + ASSERT_TRUE(strstr(err, refusal) != NULL); th_cleanup(base); th_cleanup(cache); PASS(); @@ -463,6 +497,323 @@ TEST(ws_linked_home_classified_as_home) { #endif } +/* ── Manifest entries are judged in their resolved form ──────────────────── */ + +enum { WS_T_PATH = 4096 }; + +/* th_mktempdir hands back one static buffer, so copy at once — and in canonical + * form, because the policy compares resolved spellings and /tmp is a link on + * macOS. */ +static bool ws_tempdir_canonical(const char *prefix, char *out, size_t out_sz) { + char *created = th_mktempdir(prefix); + return created && cbm_canonical_path(created, out, out_sz) == 1; +} + +static bool ws_join(char *out, size_t out_sz, const char *a, const char *b) { + int n = snprintf(out, out_sz, "%s/%s", a, b); + return n > 0 && (size_t)n < out_sz; +} + +/* Create / and return it in canonical form. */ +static bool ws_mkdir_canonical(const char *parent, const char *name, char *out, size_t out_sz) { + char made[WS_T_PATH]; + if (!ws_join(made, sizeof(made), parent, name) || cbm_mkdir(made) != 0) { + return false; + } + return cbm_canonical_path(made, out, out_sz) == 1; +} + +static void ws_manifest_write(const char *base, const char *body) { + char path[WS_T_PATH]; + if (ws_join(path, sizeof(path), base, CBM_WS_MANIFEST_NAME)) { + ws_write(path, body); + } +} + +static void ws_manifest_write_line(const char *base, const char *entry) { + char body[WS_T_PATH]; + int n = snprintf(body, sizeof(body), "%s\n", entry); + if (n > 0 && (size_t)n < sizeof(body)) { + ws_manifest_write(base, body); + } +} + +/* Put the manifest on record as approved by writing the store line directly, + * in the form cbm_workspace_manifest_approve writes (" "). + * This is the record an approval leaves behind when the entry was judged by + * its spelling, and the use-time check must stand on its own against it. */ +static bool ws_manifest_record_approval(const char *cache, const char *base) { + cbm_ws_manifest_t m; + if (!cbm_workspace_manifest_read(base, &m) || !m.present) { + return false; + } + char store[WS_T_PATH]; + char line[WS_T_PATH]; + if (!ws_join(store, sizeof(store), cache, "approved_manifests")) { + return false; + } + int n = snprintf(line, sizeof(line), "%s %s\n", m.digest, base); + if (n <= 0 || (size_t)n >= sizeof(line)) { + return false; + } + ws_write(store, line); + return cbm_workspace_manifest_is_approved(cache, base, &m); +} + +/* The directory a manifest entry names is the one that would be indexed, so + * that is the form the policy judges — at approval and again at use. Spelled + * through `..`, an entry reads as an ordinary path several components deep + * while naming the home directory or the volume root. */ +TEST(ws_manifest_entry_classified_in_resolved_form) { + char base[WS_T_PATH]; + char cache[WS_T_PATH]; + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m5", base, sizeof(base))); + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m5c", cache, sizeof(cache))); + + char home[WS_T_PATH]; + char src[WS_T_PATH]; + char home_sub[WS_T_PATH]; + ASSERT_TRUE(ws_mkdir_canonical(base, "home", home, sizeof(home))); + ASSERT_TRUE(ws_mkdir_canonical(base, "src", src, sizeof(src))); + ASSERT_TRUE(ws_mkdir_canonical(home, "notes", home_sub, sizeof(home_sub))); + + char entry[WS_T_PATH]; + char expected[WS_T_PATH]; + char err[1024]; + + /* "/src/../home" is the home directory. */ + ASSERT_TRUE(ws_join(entry, sizeof(entry), src, "../home")); + ws_manifest_write_line(base, entry); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, home, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, entry)); + ASSERT(snprintf(expected, sizeof(expected), "(resolves to %s)", home) > 0); + ASSERT_NOT_NULL(strstr(err, expected)); + ASSERT_NOT_NULL(strstr(err, cbm_workspace_verdict_reason(CBM_WS_DENY_SENSITIVE))); + + /* An approval already on record for this spelling does not outrank the + * use-time check. */ + ASSERT_TRUE(ws_manifest_record_approval(cache, base)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, home, base, home_sub)); + + /* Enough `..` segments reach the volume root from anywhere. */ + ASSERT_TRUE(ws_join(entry, sizeof(entry), base, "../../../../../../../../../../../..")); + ws_manifest_write_line(base, entry); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, home, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, entry)); + ASSERT_NOT_NULL(strstr(err, cbm_workspace_verdict_reason(CBM_WS_DENY_ABSOLUTE))); + + th_cleanup(base); + th_cleanup(cache); + PASS(); +} + +/* An entry that does not resolve to a directory is refused at approval, naming + * the entry; at use it simply matches nothing, and the other entries of the + * same manifest keep working. */ +TEST(ws_manifest_unresolvable_entry_refused) { + char base[WS_T_PATH]; + char cache[WS_T_PATH]; + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m6", base, sizeof(base))); + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m6c", cache, sizeof(cache))); + + char entry[WS_T_PATH]; + char err[1024]; + + /* Missing. */ + ASSERT_TRUE(ws_join(entry, sizeof(entry), base, "missing")); + ws_manifest_write_line(base, entry); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, entry)); + ASSERT_NOT_NULL(strstr(err, "must name an existing directory")); + + /* Present, but a file. */ + ASSERT_TRUE(ws_join(entry, sizeof(entry), base, "notes.txt")); + ws_write(entry, "not a directory\n"); + ws_manifest_write_line(base, entry); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, entry)); + ASSERT_NOT_NULL(strstr(err, "is not a directory")); + + /* Relative: the file is read by a long-lived process whose working + * directory is unrelated to the project, so there is no base to resolve + * it against. */ + ws_manifest_write_line(base, "src"); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, "requested path src:")); + ASSERT_NOT_NULL(strstr(err, "must be an absolute path")); + + /* At use: a directory that disappeared after approval matches nothing and + * does not take the rest of the manifest with it. */ + char present[WS_T_PATH]; + char present_sub[WS_T_PATH]; + char body[WS_T_PATH]; + ASSERT_TRUE(ws_mkdir_canonical(base, "present", present, sizeof(present))); + ASSERT_TRUE(ws_mkdir_canonical(present, "inc", present_sub, sizeof(present_sub))); + ASSERT_TRUE(ws_join(entry, sizeof(entry), base, "gone")); + ASSERT(snprintf(body, sizeof(body), "%s\n%s\n", entry, present) > 0); + ws_manifest_write(base, body); + ASSERT_TRUE(ws_manifest_record_approval(cache, base)); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, HOME, base, present_sub)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, HOME, base, base)); + + th_cleanup(base); + th_cleanup(cache); + PASS(); +} + +/* The ordinary shapes keep their result: an absolute existing directory, an + * entry equal to the project root, an entry outside every root in the grant + * store (the manifest is consulted on its own), and on POSIX a link to an + * ordinary directory. */ +TEST(ws_manifest_ordinary_entries_unchanged) { + char base[WS_T_PATH]; + char cache[WS_T_PATH]; + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m7", base, sizeof(base))); + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m7c", cache, sizeof(cache))); + + char sdk[WS_T_PATH]; + char sdk_inc[WS_T_PATH]; + char granted[WS_T_PATH]; + ASSERT_TRUE(ws_mkdir_canonical(base, "sdk", sdk, sizeof(sdk))); + ASSERT_TRUE(ws_mkdir_canonical(sdk, "inc", sdk_inc, sizeof(sdk_inc))); + ASSERT_TRUE(ws_mkdir_canonical(base, "granted", granted, sizeof(granted))); + + char err[1024]; + /* The grant store names a root the sdk is not under. */ + ASSERT_TRUE(cbm_workspace_grant_add(cache, HOME, granted, false, err, sizeof(err))); + + ws_manifest_write_line(base, sdk); + ASSERT_TRUE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_STR_EQ(err, ""); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, HOME, base, sdk_inc)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, HOME, base, granted)); + + /* Equal to the project root. */ + ws_manifest_write_line(base, base); + ASSERT_TRUE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, HOME, base, base)); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, HOME, base, sdk_inc)); + +#ifndef _WIN32 + /* A link to an ordinary directory is that directory. Windows is not + * covered here: symlink(2) has no counterpart in the test helpers, and + * CreateSymbolicLinkW needs a privilege the CI runner does not hold; the + * resolved-form rules themselves are exercised on every platform by the + * `..` cases above. */ + char alias[WS_T_PATH]; + ASSERT_TRUE(ws_join(alias, sizeof(alias), base, "alias")); + ASSERT_EQ(symlink(sdk, alias), 0); + ws_manifest_write_line(base, alias); + ASSERT_TRUE(cbm_workspace_manifest_approve(cache, HOME, base, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, HOME, base, sdk_inc)); +#endif + + th_cleanup(base); + th_cleanup(cache); + PASS(); +} + +#ifndef _WIN32 +/* A link is judged by where it leads. Windows is not covered: symlink(2) has + * no counterpart in the test helpers and CreateSymbolicLinkW needs a privilege + * the CI runner does not hold; the resolved-form rules are exercised on every + * platform by ws_manifest_entry_classified_in_resolved_form. */ +TEST(ws_manifest_link_to_sensitive_directory_refused) { + char base[WS_T_PATH]; + char cache[WS_T_PATH]; + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m8", base, sizeof(base))); + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m8c", cache, sizeof(cache))); + + char home[WS_T_PATH]; + char home_sub[WS_T_PATH]; + char keys[WS_T_PATH]; + char keys_sub[WS_T_PATH]; + ASSERT_TRUE(ws_mkdir_canonical(base, "home", home, sizeof(home))); + ASSERT_TRUE(ws_mkdir_canonical(home, "notes", home_sub, sizeof(home_sub))); + ASSERT_TRUE(ws_mkdir_canonical(base, ".ssh", keys, sizeof(keys))); + ASSERT_TRUE(ws_mkdir_canonical(keys, "hosts", keys_sub, sizeof(keys_sub))); + + char to_home[WS_T_PATH]; + char to_keys[WS_T_PATH]; + ASSERT_TRUE(ws_join(to_home, sizeof(to_home), base, "to-home")); + ASSERT_TRUE(ws_join(to_keys, sizeof(to_keys), base, "to-keys")); + ASSERT_EQ(symlink(home, to_home), 0); + ASSERT_EQ(symlink(keys, to_keys), 0); + + char expected[WS_T_PATH]; + char err[1024]; + + ws_manifest_write_line(base, to_home); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, home, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, to_home)); + ASSERT(snprintf(expected, sizeof(expected), "(resolves to %s)", home) > 0); + ASSERT_NOT_NULL(strstr(err, expected)); + ASSERT_NOT_NULL(strstr(err, cbm_workspace_verdict_reason(CBM_WS_DENY_SENSITIVE))); + + ws_manifest_write_line(base, to_keys); + ASSERT_FALSE(cbm_workspace_manifest_approve(cache, home, base, err, sizeof(err))); + ASSERT_NOT_NULL(strstr(err, to_keys)); + ASSERT(snprintf(expected, sizeof(expected), "(resolves to %s)", keys) > 0); + ASSERT_NOT_NULL(strstr(err, expected)); + ASSERT_NOT_NULL(strstr(err, cbm_workspace_verdict_reason(CBM_WS_DENY_SENSITIVE))); + + /* On record as approved: the use-time check still refuses both. */ + char body[WS_T_PATH]; + ASSERT(snprintf(body, sizeof(body), "%s\n%s\n", to_home, to_keys) > 0); + ws_manifest_write(base, body); + ASSERT_TRUE(ws_manifest_record_approval(cache, base)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, home, base, home_sub)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, home, base, keys_sub)); + + th_cleanup(base); + th_cleanup(cache); + PASS(); +} + +/* Approval binds to the manifest text; where a link leads is read again each + * time the manifest is used. A link approved while it led to an ordinary + * directory and later pointed at the home directory grants nothing there. + * POSIX only, for the reason given at the test above. */ +TEST(ws_manifest_link_retarget_is_seen_at_use) { + char base[WS_T_PATH]; + char cache[WS_T_PATH]; + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m9", base, sizeof(base))); + ASSERT_TRUE(ws_tempdir_canonical("cbm_ws_m9c", cache, sizeof(cache))); + + char shared[WS_T_PATH]; + char shared_sub[WS_T_PATH]; + char home[WS_T_PATH]; + char home_sub[WS_T_PATH]; + ASSERT_TRUE(ws_mkdir_canonical(base, "shared", shared, sizeof(shared))); + ASSERT_TRUE(ws_mkdir_canonical(shared, "proto", shared_sub, sizeof(shared_sub))); + ASSERT_TRUE(ws_mkdir_canonical(base, "home", home, sizeof(home))); + ASSERT_TRUE(ws_mkdir_canonical(home, "notes", home_sub, sizeof(home_sub))); + + char extra[WS_T_PATH]; + ASSERT_TRUE(ws_join(extra, sizeof(extra), base, "extra")); + ASSERT_EQ(symlink(shared, extra), 0); + + char err[1024]; + ws_manifest_write_line(base, extra); + ASSERT_TRUE(cbm_workspace_manifest_approve(cache, home, base, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_manifest_allows(cache, home, base, shared_sub)); + + /* Same text, new target. */ + ASSERT_EQ(unlink(extra), 0); + ASSERT_EQ(symlink(home, extra), 0); + cbm_ws_manifest_t m; + ASSERT_TRUE(cbm_workspace_manifest_read(base, &m)); + ASSERT_TRUE(cbm_workspace_manifest_is_approved(cache, base, &m)); + + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, home, base, home_sub)); + ASSERT_FALSE(cbm_workspace_manifest_allows(cache, home, base, shared_sub)); + + th_cleanup(base); + th_cleanup(cache); + PASS(); +} +#endif /* _WIN32 */ + SUITE(workspace) { RUN_TEST(ws_manifest_absent_is_not_an_error); RUN_TEST(ws_manifest_parses_entries_and_skips_comments); @@ -486,4 +837,11 @@ SUITE(workspace) { RUN_TEST(ws_every_verdict_has_a_reason); RUN_TEST(ws_home_dir_is_resolved); RUN_TEST(ws_linked_home_classified_as_home); + RUN_TEST(ws_manifest_entry_classified_in_resolved_form); + RUN_TEST(ws_manifest_unresolvable_entry_refused); + RUN_TEST(ws_manifest_ordinary_entries_unchanged); +#ifndef _WIN32 + RUN_TEST(ws_manifest_link_to_sensitive_directory_refused); + RUN_TEST(ws_manifest_link_retarget_is_seen_at_use); +#endif } From f7b541ee49207028d6873a2d03ad4b88ef816a57 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 2 Oct 2026 21:36:19 +0200 Subject: [PATCH 23/34] fix(mcp): search_code keeps only matches that resolve inside the root search_code returns the lines its scan opened. The scoped file list is built from the index, and an indexed path that now passes through a directory link still looks like a regular file to lstat, so grep opened and reported the link's target. Reading a result's source already went through cbm_path_within_root (attach_result_source); the raw match and the result row did not. scan_and_classify_grep_matches now applies the same check to every file in the scan stream, once per distinct file, and drops the hits of a file that resolves outside the project root. Dropped hits are not counted, like a path_filter miss. This covers the scoped and the recursive scan on both platforms. Links that stay inside the root keep working as before. Signed-off-by: Martin Vogel --- src/mcp/mcp.c | 37 ++++-- tests/test_mcp.c | 294 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 7 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 805ea7057..72f08892d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -14536,6 +14536,23 @@ static void free_file_nodes(cbm_node_t *nodes, int count) { free(nodes); } +/* True when "/" resolves inside root_path. The scan stream + * carries whatever its walker opened: the scoped list is the index's paths, + * and an indexed path that passes through a directory link is a regular file + * to lstat, so grep reads the link's target; the recursive walker is the + * platform's own. This is the containment check attach_result_source already + * applies before it reads a result's source, applied to the match itself, + * once per distinct file (grep groups a file's hits). A path too long for + * the canonical-path buffers is one that check cannot resolve either. */ +static bool search_hit_file_within_root(const char *root_path, const char *file) { + char abs_path[CBM_SZ_4K]; + int length = snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, file); + if (length < 0 || (size_t)length >= sizeof(abs_path)) { + return false; + } + return cbm_path_within_root(root_path, abs_path); +} + /* Parse and classify the complete grep stream without retaining one object per * hit. Graph results retain one identity per distinct node plus at most 500 * line numbers each; raw matches retain only the caller's requested page. @@ -14549,6 +14566,7 @@ static bool scan_and_classify_grep_matches( char *line = NULL; size_t line_capacity = 0; char *current_file = NULL; + bool current_file_inside = false; cbm_node_t *file_nodes = NULL; int file_node_count = 0; cbm_regex_t content_regex; @@ -14594,12 +14612,6 @@ static bool scan_and_classify_grep_matches( if (has_path_filter && cbm_regexec(path_regex, file, 0, NULL, 0) != CBM_REG_OK) { continue; } - if (*grep_count == INT_MAX) { - ok = false; - break; - } - (*grep_count)++; - if (!current_file || strcmp(current_file, file) != 0) { free_file_nodes(file_nodes, file_node_count); file_nodes = NULL; @@ -14610,11 +14622,22 @@ static bool scan_and_classify_grep_matches( ok = false; break; } - if (store) { + current_file_inside = search_hit_file_within_root(root_path, file); + if (store && current_file_inside) { (void)cbm_store_find_nodes_by_file(store, project, file, &file_nodes, &file_node_count); } } + if (!current_file_inside) { + /* Every hit of a file that resolves outside the root is dropped + * and, like a path_filter miss, not counted. */ + continue; + } + if (*grep_count == INT_MAX) { + ok = false; + break; + } + (*grep_count)++; grep_match_t hit = {0}; hit.file = heap_strdup(file); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 4c7558fee..57cc81a8f 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -22230,6 +22230,296 @@ TEST(search_code_rejects_quote_in_file_pattern) { PASS(); } +/* ── search_code: every match resolves inside the project root ───────── + * search_code returns the lines its scan opened. The scoped file list is + * built from the index, and an indexed path that now passes through a + * directory link still looks like a regular file to lstat, so grep opens + * and reports the link target. Reading a result's source already goes + * through cbm_path_within_root (attach_result_source); the match itself + * must pass the same check in every output mode. + * + * POSIX only: the fixture needs symbolic links. Windows: creating a link + * or junction from a test needs CreateSymbolicLinkW (a privilege unless + * developer mode is on) or a junction helper the test tree does not have, + * and a Windows run of these cases was not attempted in this change, so + * they skip with that reason rather than sit as an unexplained red. */ +#ifndef _WIN32 +typedef struct { + char root[CBM_SZ_512]; + char outside[CBM_SZ_512]; +} search_root_fixture_t; + +/* Layout (every line that matters carries ROOTCHECK_NEEDLE): + * /main.c ordinary file, Variable INSIDE_SYMBOL + * /inner/same.c ordinary file, reached via the links + * /alias -> inner directory link that stays inside + * /mirror.c -> inner/same.c file link that stays inside + * /link.c -> /far.c file link that leaves the root + * /lib -> /dir directory link that leaves the root + * /dir/deep.c raw hit through the directory link + * /dir/sym.c graph hit (Variable FAR_DIR_SYMBOL) */ +static bool search_root_fixture_create(search_root_fixture_t *fx) { + snprintf(fx->root, sizeof(fx->root), "%s/cbm_srch_root_XXXXXX", cbm_tmpdir()); + snprintf(fx->outside, sizeof(fx->outside), "%s/cbm_srch_outside_XXXXXX", cbm_tmpdir()); + if (!cbm_mkdtemp(fx->root) || !cbm_mkdtemp(fx->outside)) { + return false; + } + char path[CBM_SZ_1K]; + char target[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s/main.c", fx->root); + if (th_write_file(path, "int ROOTCHECK_NEEDLE_INSIDE = 1;\n") != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/inner/same.c", fx->root); + if (th_write_file(path, "int ROOTCHECK_NEEDLE_INNER = 4;\n") != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/alias", fx->root); + if (symlink("inner", path) != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/mirror.c", fx->root); + if (symlink("inner/same.c", path) != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/far.c", fx->outside); + if (th_write_file(path, "int ROOTCHECK_NEEDLE_FAR_FILE = 2;\n") != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/dir/deep.c", fx->outside); + if (th_write_file(path, "int ROOTCHECK_NEEDLE_FAR_DIR = 3;\n") != 0) { + return false; + } + snprintf(path, sizeof(path), "%s/dir/sym.c", fx->outside); + if (th_write_file(path, "int ROOTCHECK_NEEDLE_FAR_SYM = 5;\n") != 0) { + return false; + } + snprintf(target, sizeof(target), "%s/far.c", fx->outside); + snprintf(path, sizeof(path), "%s/link.c", fx->root); + if (symlink(target, path) != 0) { + return false; + } + snprintf(target, sizeof(target), "%s/dir", fx->outside); + snprintf(path, sizeof(path), "%s/lib", fx->root); + return symlink(target, path) == 0; +} + +static void search_root_fixture_destroy(search_root_fixture_t *fx) { + /* Unlink the links first so the tree removal never descends through one. */ + static const char *const links[] = {"alias", "mirror.c", "link.c", "lib"}; + for (size_t i = 0; i < sizeof(links) / sizeof(links[0]); i++) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s/%s", fx->root, links[i]); + (void)unlink(path); + } + (void)th_rmtree(fx->root); + (void)th_rmtree(fx->outside); +} + +/* File nodes as a stale index holds them: the link targets were ordinary + * directories and files when they were indexed (discovery itself never + * follows a link). inner/same.c is deliberately absent so its line can only + * be reached through alias/ or mirror.c. */ +static bool search_root_fixture_index(cbm_store_t *store, const char *project) { + static const char *const files[] = {"main.c", "alias/same.c", "mirror.c", + "link.c", "lib/deep.c", "lib/sym.c"}; + for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) { + const char *base = strrchr(files[i], '/'); + base = base ? base + 1 : files[i]; + char qualified_name[CBM_SZ_256]; + snprintf(qualified_name, sizeof(qualified_name), "%s.%s", project, files[i]); + cbm_node_t file = {.project = project, + .label = "File", + .name = base, + .qualified_name = qualified_name, + .file_path = files[i]}; + if (cbm_store_upsert_node(store, &file) <= 0) { + return false; + } + } + cbm_node_t inside = {.project = project, + .label = "Variable", + .name = "INSIDE_SYMBOL", + .qualified_name = "search-root.INSIDE_SYMBOL", + .file_path = "main.c", + .start_line = 1, + .end_line = 1}; + cbm_node_t far_symbol = {.project = project, + .label = "Variable", + .name = "FAR_DIR_SYMBOL", + .qualified_name = "search-root.lib.FAR_DIR_SYMBOL", + .file_path = "lib/sym.c", + .start_line = 1, + .end_line = 1}; + return cbm_store_upsert_node(store, &inside) > 0 && + cbm_store_upsert_node(store, &far_symbol) > 0; +} + +typedef struct { + bool ok; /* a non-error response with text content */ + bool inside_file; /* main.c reported */ + bool inside_dir_link; /* alias/same.c reported (directory link inside) */ + bool inside_file_link; /* mirror.c reported (file link inside) */ + bool outside; /* any line or path from beyond the root */ +} search_root_probe_t; + +static search_root_probe_t search_root_probe(cbm_mcp_server_t *srv, const char *args) { + search_root_probe_t probe = {0}; + char *response = cbm_mcp_handle_tool(srv, "search_code", args); + char *inner = response ? extract_text_content(response) : NULL; + probe.ok = response && !strstr(response, "\"isError\":true") && inner; + if (probe.ok) { + probe.inside_file = strstr(inner, "main.c") != NULL; + probe.inside_dir_link = strstr(inner, "alias/same.c") != NULL; + probe.inside_file_link = strstr(inner, "mirror.c") != NULL; + static const char *const outside_markers[] = {"ROOTCHECK_NEEDLE_FAR", "FAR_DIR_SYMBOL", + "deep.c", "sym.c", "link.c"}; + for (size_t i = 0; i < sizeof(outside_markers) / sizeof(outside_markers[0]); i++) { + if (strstr(inner, outside_markers[i])) { + probe.outside = true; + } + } + } + free(inner); + free(response); + return probe; +} + +#define SEARCH_ROOT_ARGS(extra) \ + "{\"pattern\":\"ROOTCHECK_NEEDLE\",\"project\":\"search-root\"" extra "}" +#endif + +/* Scoped route, real scan command: the index lists lib/deep.c and lib/sym.c, + * lstat follows the lib -> /dir link and calls both regular, grep + * reads the targets. Every output mode must leave those lines out while + * main.c and alias/same.c (a directory link that stays inside) stay in. + * mirror.c and link.c are links as their last path component; the scoped + * list writer already leaves those out (lstat reports the link itself) — + * pinned here as the existing behaviour, not changed. */ +TEST(search_code_matches_stay_inside_project_root) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX symbolic-link fixture; see search_root_fixture_create"); +#else + search_root_fixture_t fx; + bool fixture_ok = search_root_fixture_create(&fx); + if (!fixture_ok) { + search_root_fixture_destroy(&fx); + } + ASSERT_TRUE(fixture_ok); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + const char *project = "search-root"; + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_upsert_project(store, project, fx.root), CBM_STORE_OK); + ASSERT_TRUE(search_root_fixture_index(store, project)); + + static const char *const args[] = { + SEARCH_ROOT_ARGS(""), + SEARCH_ROOT_ARGS(",\"format\":\"json\""), + SEARCH_ROOT_ARGS(",\"mode\":\"full\""), + SEARCH_ROOT_ARGS(",\"mode\":\"full\",\"format\":\"json\""), + SEARCH_ROOT_ARGS(",\"mode\":\"files\""), + SEARCH_ROOT_ARGS(",\"context\":2"), + "{\"pattern\":\"ROOTCHECK_NEEDLE_[A-Z]+\",\"project\":\"search-root\",\"regex\":true}", + }; + enum { PROBES = sizeof(args) / sizeof(args[0]) }; + search_root_probe_t probes[PROBES]; + for (size_t i = 0; i < PROBES; i++) { + probes[i] = search_root_probe(srv, args[i]); + } + /* Dropped files are not counted either: the exact total is the two + * lines inside the root, as it would be after a fresh index. */ + char *json_response = cbm_mcp_handle_tool(srv, "search_code", args[1]); + char *json_inner = json_response ? extract_text_content(json_response) : NULL; + bool total_counts_inside_only = json_inner && strstr(json_inner, "\"total_grep_matches\":2"); + free(json_inner); + free(json_response); + + cbm_mcp_server_free(srv); + search_root_fixture_destroy(&fx); + + for (size_t i = 0; i < PROBES; i++) { + ASSERT_TRUE(probes[i].ok); + ASSERT_FALSE(probes[i].outside); + ASSERT_TRUE(probes[i].inside_file); + ASSERT_TRUE(probes[i].inside_dir_link); + ASSERT_FALSE(probes[i].inside_file_link); + } + ASSERT_TRUE(total_counts_inside_only); + PASS(); +#endif +} + +/* Recursive route: with no indexed files the scan walks the tree. On POSIX + * that walker is `find -type f`, which never hands grep a link, so the scan + * command is replaced by one that does — the hits a walker that follows + * links (Get-ChildItem on Windows is one) would deliver. Lines from + * link.c and lib/ resolve outside the root and must be dropped; mirror.c + * and alias/same.c resolve inside and must be kept. */ +TEST(search_code_drops_walker_hits_that_resolve_outside_root) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX symbolic-link fixture; see search_root_fixture_create"); +#else + search_root_fixture_t fx; + bool fixture_ok = search_root_fixture_create(&fx); + if (!fixture_ok) { + search_root_fixture_destroy(&fx); + } + ASSERT_TRUE(fixture_ok); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + const char *project = "search-root"; + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_upsert_project(store, project, fx.root), CBM_STORE_OK); + + char scan_command[CBM_SZ_4K]; + int command_length = + snprintf(scan_command, sizeof(scan_command), + "grep -Hn -F ROOTCHECK_NEEDLE -- '%s/main.c' '%s/mirror.c' '%s/link.c' " + "'%s/lib/deep.c' '%s/lib/sym.c' '%s/alias/same.c'", + fx.root, fx.root, fx.root, fx.root, fx.root, fx.root); + ASSERT_TRUE(command_length > 0 && (size_t)command_length < sizeof(scan_command)); + cbm_mcp_server_set_search_scan_command_for_test(srv, scan_command); + + static const char *const args[] = { + SEARCH_ROOT_ARGS(",\"raw_limit\":10"), + SEARCH_ROOT_ARGS(",\"raw_limit\":10,\"format\":\"json\""), + SEARCH_ROOT_ARGS(",\"raw_limit\":10,\"mode\":\"full\""), + SEARCH_ROOT_ARGS(",\"raw_limit\":10,\"mode\":\"files\""), + }; + enum { PROBES = sizeof(args) / sizeof(args[0]) }; + search_root_probe_t probes[PROBES]; + for (size_t i = 0; i < PROBES; i++) { + probes[i] = search_root_probe(srv, args[i]); + } + char *json_response = cbm_mcp_handle_tool(srv, "search_code", args[1]); + cbm_mcp_server_set_search_scan_command_for_test(srv, NULL); + char *json_inner = json_response ? extract_text_content(json_response) : NULL; + bool total_counts_inside_only = json_inner && strstr(json_inner, "\"total_grep_matches\":3"); + free(json_inner); + free(json_response); + + cbm_mcp_server_free(srv); + search_root_fixture_destroy(&fx); + + for (size_t i = 0; i < PROBES; i++) { + ASSERT_TRUE(probes[i].ok); + ASSERT_FALSE(probes[i].outside); + ASSERT_TRUE(probes[i].inside_file); + ASSERT_TRUE(probes[i].inside_dir_link); + ASSERT_TRUE(probes[i].inside_file_link); + } + ASSERT_TRUE(total_counts_inside_only); + PASS(); +#endif +} + SUITE(mcp) { RUN_TEST(index_repository_async_and_status_refused_without_daemon_issue2144); RUN_TEST(index_repository_schema_documents_async_polling_issue2144); @@ -22627,6 +22917,10 @@ SUITE(mcp) { RUN_TEST(tool_search_graph_limit_above_ceiling_is_capped); RUN_TEST(tool_trace_cursor_truncated_after_leg_is_rejected); RUN_TEST(search_code_rejects_quote_in_file_pattern); + + /* search_code: every match resolves inside the project root */ + RUN_TEST(search_code_matches_stay_inside_project_root); + RUN_TEST(search_code_drops_walker_hits_that_resolve_outside_root); } /* Kept separate so daemon-coordination regressions can be iterated without From 4e490bef5af969a8f31a2cceda10a665f9f741d0 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 28 Sep 2026 19:34:20 +0200 Subject: [PATCH 24/34] fix(routes): HANDLES for Laravel array and invokable controller actions Why: Laravel names a route's controller as [UserController::class, 'show'], or, for an invokable controller, as GetCurrentUserController::class. The route handler scan only took identifiers and strings, so these routes never got a HANDLES edge. The cross-repo HTTP matcher reaches a route's handler through that edge, so apps written this way - the #1146 reporter's app among them - produced 0 CROSS_HTTP_CALLS. Resolving the class by short name is not enough either: the call pass's import map bound `use App\Http\Controllers\UserController` to Admin\UserController by name, even with a composer.json PSR-4 map present, and the generic resolver's multi-candidate fallback bound a vendor controller's show() to another class's show(). Fix: - extract_calls.c: for PHP, [X::class, 'm'] becomes the handler reference "Ns\X::m" and X::class becomes "Ns\X::__invoke". X is qualified the way PHP resolves it: through the file's `use` imports (aliases included), else the file namespace; a leading backslash is already absolute. - registry.c, cbm_registry_resolve_handler (both route passes): a class-qualified handler is placed on a method QN only by what places the class file - the composer.json PSR-4 root in the package map, else the one candidate whose folders mirror the namespace (App\Http\Controllers -> app/Http/Controllers). A class that is not in the repo gets no handler. A handler reference without "::" resolves exactly as before. Proof (release CLI built from this tree vs origin/main 1f9b4db1): - laravel.io: HANDLES 0 -> 4, one for each of the 5 route registrations main mints (two share a Route). The use statement tells Articles\ArticlesController apart from Admin\ArticlesController. Route count unchanged (39). With the slashless-path and withRouting fixes also applied: HANDLES 0 -> 58, all from PHP controllers, samples checked against source (aliased AdminArticlesController and AdminUsersController included); the 7 routes left without a handler use methods from a vendor trait (AuthenticatesUsers). - Reporter-shaped cross-repo check (Laravel 11 withRouting api:, invokable GetCurrentUserController under prefix('/users'), TS fetch('/api/users/me')): CROSS_HTTP_CALLS 0 with this fix alone (route /users/me, no /api mount) and 0 with the withRouting fix alone (no HANDLES); 2 with both applied. - krayin/laravel-crm (reduced to one route file): / <- Webkul\Admin\Http\ Controllers\Controller::redirectToLogin, placed through the PSR-4 root Webkul\Admin\ -> packages/Webkul/Admin/src. Tests (edge_types_probe): handles_laravel_class_handlers_issue1146, handles_laravel_class_handlers_parallel_issue1146 and handles_laravel_class_handlers_no_composer_issue1146 assert the exact HANDLES set. They include controls for a same-named Admin\UserController::show, an alias import, an unimported fully-qualified class, a package under a PSR-4 root whose folder does not mirror its namespace, a decoy __invoke, and two vendor controllers that must get no edge. RED on origin/main 3x, green, RED on revert. edge_types_probe, php_lsp, pipeline, parallel, route_canon and cross_repo pass (730); make lint-ci passes. Refs #1146 Signed-off-by: Martin Vogel --- internal/cbm/extract_calls.c | 124 +++++++++++++++++++++++ src/pipeline/pass_calls.c | 5 +- src/pipeline/pass_parallel.c | 3 +- src/pipeline/pipeline_internal.h | 14 +++ src/pipeline/registry.c | 168 +++++++++++++++++++++++++++++++ tests/test_edge_types_probe.c | 166 ++++++++++++++++++++++++++++++ 6 files changed, 477 insertions(+), 3 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index f14efb766..4da0ece7a 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2553,6 +2553,123 @@ static const char *normalize_string_handler(CBMArena *a, const char *raw) { return unq; } +/* True for a PHP class/namespace name: identifier segments joined by '\'. */ +static bool php_is_class_path(const char *s) { + if (!s || !s[0]) { + return false; + } + for (; *s; s++) { + if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || (*s >= '0' && *s <= '9') || + *s == '_' || *s == '\\')) { + return false; + } + } + return true; +} + +/* Qualify a class name the way PHP resolves it: a leading '\' is already + * absolute; a first segment that a `use` clause imports (by alias or by last + * segment) is replaced by the imported name; anything else is relative to the + * file's namespace. Imports and the namespace are extracted before the call + * walk, so both are complete here. */ +static const char *php_qualify_class(CBMExtractCtx *ctx, const char *written) { + if (written[0] == '\\') { + return written + SKIP_ONE; + } + const char *sep = strchr(written, '\\'); + size_t first_len = sep ? (size_t)(sep - written) : strlen(written); + const CBMImportArray *imps = &ctx->result->imports; + for (int i = 0; i < imps->count; i++) { + const CBMImport *imp = &imps->items[i]; + const char *path = imp->module_path; + if (!imp->local_name || !path || strlen(imp->local_name) != first_len || + strncmp(imp->local_name, written, first_len) != 0) { + continue; + } + path += path[0] == '\\' ? SKIP_ONE : 0; + if (php_is_class_path(path)) { + return cbm_arena_sprintf(ctx->arena, "%s%s", path, sep ? sep : ""); + } + } + const char *ns = ctx->result->namespace_name; + return ns && ns[0] ? cbm_arena_sprintf(ctx->arena, "%s\\%s", ns, written) : written; +} + +/* The fully-qualified class a PHP `X::class` expression names (`use` imports + * and the file namespace applied). NULL for any other expression, including + * `static::class` and `self::class`. */ +static const char *php_class_literal_fqn(CBMExtractCtx *ctx, TSNode node) { + if (strcmp(ts_node_type(node), "class_constant_access_expression") != 0 || + ts_node_named_child_count(node) != PAIR_LEN) { + return NULL; + } + TSNode cls = ts_node_named_child(node, 0); + const char *ck = ts_node_type(cls); + const char *member = + cbm_node_text(ctx->arena, ts_node_named_child(node, SKIP_ONE), ctx->source); + if ((strcmp(ck, "name") != 0 && strcmp(ck, "qualified_name") != 0) || !member || + strcmp(member, "class") != 0) { + return NULL; + } + const char *text = cbm_node_text(ctx->arena, cls, ctx->source); + return php_is_class_path(text) ? php_qualify_class(ctx, text) : NULL; +} + +/* The value of a PHP array element written without a key; a null node for a + * keyed element (`'uses' => ...`) or anything else. */ +static TSNode php_unkeyed_element_value(TSNode elem) { + TSNode none = {0}; + if (strcmp(ts_node_type(elem), "array_element_initializer") != 0 || + ts_node_named_child_count(elem) != SKIP_ONE) { + return none; + } + return ts_node_named_child(elem, 0); +} + +static bool php_is_identifier(const char *s) { + if (!s || !((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || *s == '_')) { + return false; + } + for (; *s; s++) { + if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || (*s >= '0' && *s <= '9') || + *s == '_')) { + return false; + } + } + return true; +} + +/* Laravel's class-based route handlers (#1146), as "Ns\Class::method": + * [UserController::class, 'show'] -> App\Http\Controllers\UserController::show + * GetCurrentUserController::class -> App\...\GetCurrentUserController::__invoke + * with the class qualified through the file's `use` imports; the route passes + * place it on a method QN (cbm_registry_resolve_handler). NULL for any other + * expression. */ +static const char *php_class_route_handler(CBMExtractCtx *ctx, TSNode arg) { + const char *kind = ts_node_type(arg); + if (strcmp(kind, "class_constant_access_expression") == 0) { + const char *cls = php_class_literal_fqn(ctx, arg); + return cls ? cbm_arena_sprintf(ctx->arena, "%s::__invoke", cls) : NULL; + } + if (strcmp(kind, "array_creation_expression") != 0 || + ts_node_named_child_count(arg) != PAIR_LEN) { + return NULL; + } + TSNode cls_node = php_unkeyed_element_value(ts_node_named_child(arg, 0)); + TSNode method_node = php_unkeyed_element_value(ts_node_named_child(arg, SKIP_ONE)); + if (ts_node_is_null(cls_node) || ts_node_is_null(method_node)) { + return NULL; + } + const char *cls = php_class_literal_fqn(ctx, cls_node); + const char *mk = ts_node_type(method_node); + if (!cls || (strcmp(mk, "string") != 0 && strcmp(mk, "encapsed_string") != 0)) { + return NULL; + } + const char *method = + strip_quotes(ctx->arena, cbm_node_text(ctx->arena, method_node, ctx->source)); + return php_is_identifier(method) ? cbm_arena_sprintf(ctx->arena, "%s::%s", cls, method) : NULL; +} + static const char *extract_handler_arg(CBMExtractCtx *ctx, TSNode args) { /* The LAST eligible argument wins, and every argument is examined. * Express, Fastify, gin and Laravel all put middleware between the route @@ -2570,6 +2687,13 @@ static const char *extract_handler_arg(CBMExtractCtx *ctx, TSNode args) { if (strcmp(ts_node_type(arg2), "argument") == 0 && ts_node_named_child_count(arg2) > 0) { arg2 = ts_node_named_child(arg2, 0); } + if (ctx->language == CBM_LANG_PHP) { + const char *class_handler = php_class_route_handler(ctx, arg2); + if (class_handler) { + handler = class_handler; + continue; + } + } const char *ak2 = ts_node_type(arg2); /* `name` = PHP bare identifier handler; string = Laravel string handler * ('showUsers' or 'Controller@method'). */ diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 668919682..53f57025a 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -404,8 +404,9 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca esc_fa); cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, - module_qn, imp_keys, imp_vals, imp_count); + cbm_resolution_t hres = + cbm_registry_resolve_handler(ctx->registry, call->second_arg_name, module_qn, imp_keys, + imp_vals, imp_count, cbm_pipeline_get_pkgmap()); if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *handler = cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); if (handler != NULL) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a8a9f3693..dfaea7161 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2295,7 +2295,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou esc_rp); cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); + cbm_resolution_t hres = cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, + iv, ic, cbm_pipeline_get_pkgmap()); if (hres.qualified_name && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); if (h) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae9..451776807 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -208,6 +208,20 @@ static inline int cbm_pipeline_relpath_is_excluded(const char *rel_path, char *c CBMHashTable *cbm_pipeline_get_pkgmap(void); void cbm_pipeline_set_pkgmap(CBMHashTable *map); +/* Resolve a route handler reference (the registry's answer for route passes). + * A class-qualified "Ns\\Class::method" reference — the extractor's form for + * Laravel `[Class::class, 'method']` and invokable `Class::class` handlers + * (#1146), with the class already qualified through the file's `use` imports + * and namespace — is placed on a method QN only by what places the class file: + * a composer.json PSR-4 root in psr4_roots (the package map), else the one + * candidate whose directories end with the class's namespace. Never by short + * name alone, so a vendor controller that is not in the repo gets no handler. + * A reference without "::" resolves exactly as cbm_registry_resolve does. */ +cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + const CBMHashTable *psr4_roots); + /* Unified module resolver: relative → pkgmap → fqn_module fallback. * Handles bare specifiers via pkgmap lookup with prefix matching. * Caller must free() the returned string. */ diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index ff56a41ac..4159c19f4 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -25,6 +25,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #define DEFAULT_CONFIDENCE 0.5 #include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" #include "cbm.h" /* cbm_label_is_relation — the resolve-time relation veto */ #include "foundation/compat.h" /* CBM_TLS */ #include "foundation/hash_table.h" @@ -1462,6 +1463,173 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle return res; } +/* ── Class-qualified route handlers (PHP, #1146) ─────────────────── */ + +/* Method QNs are path-based: ..... + * A PHP class is named by its namespace instead, so a "Ns\\Class::method" + * handler is placed on a QN by what places the class file itself. */ +#define CONF_PSR4_MEMBER 0.95 +#define CONF_NAMESPACE_PATH_MEMBER 0.90 + +static bool seg_eq_nocase(const char *a, size_t alen, const char *b, size_t blen) { + if (alen != blen) { + return false; + } + for (size_t i = 0; i < alen; i++) { + char ca = (char)((a[i] >= 'A' && a[i] <= 'Z') ? a[i] - 'A' + 'a' : a[i]); + char cb = (char)((b[i] >= 'A' && b[i] <= 'Z') ? b[i] - 'A' + 'a' : b[i]); + if (ca != cb) { + return false; + } + } + return true; +} + +/* Does qn end with ".."? Returns the offset of that tail or -1. */ +static int qn_class_member_tail(const char *qn, const char *cls, size_t cls_len, + const char *member) { + size_t qlen = strlen(qn); + size_t mlen = strlen(member); + if (qlen < cls_len + mlen + PAIR_LEN) { + return CBM_NOT_FOUND; + } + const char *m = qn + qlen - mlen; + const char *c = m - SKIP_ONE - cls_len; + if (strcmp(m, member) != 0 || m[-SKIP_ONE] != '.' || strncmp(c, cls, cls_len) != 0 || + c[-SKIP_ONE] != '.') { + return CBM_NOT_FOUND; + } + return (int)(c - SKIP_ONE - qn); +} + +/* PSR-4: the longest composer.json autoload root "Prefix\\" that prefixes the + * FQN maps the remaining namespace onto directories below the root's folder + * and the class onto .php — the file Composer's autoloader loads. + * Returns the registered method QN, or NULL (no root prefixes the FQN, or the + * autoloaded file holds no such method). */ +static const char *psr4_member_qn(const cbm_registry_t *r, const CBMHashTable *psr4_roots, + const char *fqn, size_t fqn_len, const char *cls, size_t cls_len, + const char *member) { + if (!psr4_roots || fqn_len >= CBM_SZ_512) { + return NULL; + } + char key[CBM_SZ_512]; + for (size_t cut = fqn_len; cut > 0; cut--) { + if (fqn[cut - SKIP_ONE] != '\\') { + continue; + } + memcpy(key, fqn, cut); /* "App\\" / "App\\Http\\" — keeps the trailing '\\' */ + key[cut] = '\0'; + const char *root_qn = (const char *)cbm_ht_get(psr4_roots, key); + if (!root_qn) { + continue; + } + char qn[CBM_SZ_1K]; + int n = snprintf(qn, sizeof(qn), "%s.%.*s.%.*s.%s", root_qn, (int)(fqn_len - cut), + fqn + cut, (int)cls_len, cls, member); + if (n <= 0 || (size_t)n >= sizeof(qn)) { + return NULL; + } + for (char *p = qn + strlen(root_qn); *p; p++) { + if (*p == '\\') { + *p = '.'; + } + } + return cbm_ht_get_key(r->exact, qn); + } + return NULL; +} + +/* Do the directory segments of qn (between the project root segment and the + * file stem, i.e. before offset tail = "..") end with every + * segment of the namespace fqn[0, ns_len)? Case-insensitive: App\\Http is + * conventionally app/Http. An empty namespace is trivially aligned. */ +static bool namespace_matches_dirs(const char *qn, int tail, const char *fqn, size_t ns_len) { + const char *root_dot = strchr(qn, '.'); + const char *stem_end = qn + tail; + if (!root_dot || stem_end <= root_dot) { + return false; + } + const char *dir_end = stem_end - SKIP_ONE; /* back over the file stem */ + while (*dir_end != '.') { + dir_end--; + } + size_t ns_end = ns_len; + while (ns_end > 0) { + size_t ns_start = ns_end; + while (ns_start > 0 && fqn[ns_start - SKIP_ONE] != '\\') { + ns_start--; + } + if (dir_end <= root_dot) { + return false; /* more namespace segments than directories */ + } + const char *dir_start = dir_end; + while (dir_start[-SKIP_ONE] != '.') { + dir_start--; + } + if (!seg_eq_nocase(fqn + ns_start, ns_end - ns_start, dir_start, + (size_t)(dir_end - dir_start))) { + return false; + } + dir_end = dir_start - SKIP_ONE; + ns_end = ns_start > 0 ? ns_start - SKIP_ONE : 0; + } + return true; +} + +/* Namespace/path alignment: the one "." candidate that sits + * where its namespace says. NULL when none or several do. */ +static const char *aligned_member_qn(const cbm_registry_t *r, const char *fqn, size_t ns_len, + const char *cls, size_t cls_len, const char *member) { + const qn_array_t *arr = cbm_ht_get(r->by_name, member); + const char *match = NULL; + for (int i = 0; arr && i < arr->count; i++) { + const char *qn = arr->items[i]; + int tail = qn_class_member_tail(qn, cls, cls_len, member); + if (tail == CBM_NOT_FOUND || !namespace_matches_dirs(qn, tail, fqn, ns_len)) { + continue; + } + if (match) { + return NULL; /* ambiguous */ + } + match = qn; + } + return match; +} + +cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count, + const CBMHashTable *psr4_roots) { + const char *sep = handler_ref ? strstr(handler_ref, "::") : NULL; + if (!r || !sep) { + return cbm_registry_resolve(r, handler_ref, module_qn, import_map_keys, import_map_vals, + import_map_count); + } + const char *member = sep + PAIR_LEN; + size_t fqn_len = (size_t)(sep - handler_ref); + size_t ns_len = fqn_len; + while (ns_len > 0 && handler_ref[ns_len - SKIP_ONE] != '\\') { + ns_len--; + } + const char *cls = handler_ref + ns_len; + size_t cls_len = fqn_len - ns_len; + ns_len = ns_len > 0 ? ns_len - SKIP_ONE : 0; /* drop the separator before the class */ + if (cls_len == 0 || !member[0]) { + return empty_result(); + } + const char *qn = psr4_member_qn(r, psr4_roots, handler_ref, fqn_len, cls, cls_len, member); + if (qn) { + return (cbm_resolution_t){qn, "php_psr4", CONF_PSR4_MEMBER, REG_RESOLVED}; + } + qn = aligned_member_qn(r, handler_ref, ns_len, cls, cls_len, member); + if (qn) { + return (cbm_resolution_t){qn, "php_namespace_path", CONF_NAMESPACE_PATH_MEMBER, + REG_RESOLVED}; + } + return empty_result(); +} + cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count) { diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index 5ed449b8d..999019c45 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -922,6 +922,169 @@ TEST(routes_laravel_withrouting_parallel_subdir_issue1146) { PASS(); } +/* Assert the exact HANDLES edge set as " -> " + * pairs. The tail is matched at a segment boundary against the handler's + * qualified name ("UserController.show" matches + * ".app.Http.Controllers.UserController.UserController.show"), so a + * same-named method on another class never satisfies it; the total count must + * match too, so a fabricated extra edge fails the assertion. */ +typedef struct { + const char *handler_tail; + const char *route; +} EtHandles; + +static int et_qn_has_tail(const char *qn, const char *tail) { + size_t ql = qn ? strlen(qn) : 0; + size_t tl = strlen(tail); + return ql >= tl && strcmp(qn + ql - tl, tail) == 0 && (ql == tl || qn[ql - tl - 1] == '.'); +} + +static int et_handles_exact(const EtFile *files, int nfiles, const EtHandles *want, int parallel) { + EtProj lp; + cbm_store_t *store = + parallel ? et_index_parallel(&lp, files, nfiles) : et_index_files(&lp, files, nfiles); + int wanted = 0; + int found[ET_ROUTE_ASSERT_MAX] = {0}; + while (want[wanted].handler_tail && wanted < ET_ROUTE_ASSERT_MAX) { + wanted++; + } + cbm_edge_t *edges = NULL; + int n = 0; + int ok = store != NULL && + cbm_store_find_edges_by_type(store, lp.project, "HANDLES", &edges, &n) == + CBM_STORE_OK && + n == wanted; + for (int i = 0; store && i < n; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + int have_src = cbm_store_find_node_by_id(store, edges[i].source_id, &src) == CBM_STORE_OK; + int have_tgt = cbm_store_find_node_by_id(store, edges[i].target_id, &tgt) == CBM_STORE_OK; + int matched = 0; + for (int wi = 0; have_src && have_tgt && wi < wanted; wi++) { + if (!found[wi] && tgt.name && strcmp(tgt.name, want[wi].route) == 0 && + et_qn_has_tail(src.qualified_name, want[wi].handler_tail)) { + found[wi] = matched = 1; + break; + } + } + if (!matched) { + ok = 0; + fprintf(stderr, " [ET-HANDLES] unexpected %s -> %s\n", + have_src && src.qualified_name ? src.qualified_name : "", + have_tgt && tgt.name ? tgt.name : ""); + } + if (have_src) cbm_node_free_fields(&src); + if (have_tgt) cbm_node_free_fields(&tgt); + } + for (int wi = 0; wi < wanted; wi++) { + if (!found[wi]) { + ok = 0; + fprintf(stderr, " [ET-HANDLES] missing %s -> %s\n", want[wi].handler_tail, + want[wi].route); + } + } + if (!ok) { + fprintf(stderr, " [ET-HANDLES] FAIL (%s path) expected=%d actual=%d\n", + parallel ? "parallel" : "sequential", wanted, n); + } + if (edges) cbm_store_free_edges(edges, n); + et_cleanup(&lp, store); + return ok; +} + +/* #1146: Laravel's two class-based handler forms — + * Route::get('/x', [UserController::class, 'show']) (controller method) + * Route::get('/x', GetCurrentUserController::class) (invokable: __invoke) + * got no HANDLES edge, because the handler scan only took identifiers and + * strings. The cross-repo matcher needs HANDLES on the Route, so apps written + * this way (the #1146 reporter's included) produced 0 CROSS_HTTP_CALLS. + * Controls: an Admin\UserController::show with the same short name (the `use` + * statement picks the class), an alias import, a fully-qualified class that is + * not imported, a package class under a PSR-4 root whose folder does not + * mirror its namespace (Acme\Blog\ -> packages/blog/src/), a decoy __invoke on + * another class, and two vendor classes that are not in the repo — those must + * get no HANDLES edge at all rather than bind to some other class's + * same-named method. composer.json comes first so the no-composer variant + * below can index the same fixture without it. */ +static const EtFile et_laravel_class_handlers[] = { + {"composer.json", "{\"autoload\": {\"psr-4\": {\"App\\\\\": \"app/\", " + "\"Acme\\\\Blog\\\\\": \"packages/blog/src/\"}}}\n"}, + {"packages/blog/src/Http/PostsController.php", + " []]; }\n}\n"}, + {"app/Http/Controllers/UserController.php", + " $id]; }\n" + " public function index() { return []; }\n}\n"}, + {"app/Http/Controllers/Admin/UserController.php", + " $id]; }\n}\n"}, + {"app/Http/Controllers/PostController.php", + " $id]; }\n}\n"}, + {"app/Http/Controllers/GetCurrentUserController.php", + " true]; }\n}\n"}, + {"app/Http/Controllers/HealthController.php", + " true]; }\n}\n"}, + {"routes/api.php", + " MIN_FILES_FOR_PARALLEL), + * whose route emitter (pass_parallel.c) is a separate code path. */ +TEST(handles_laravel_class_handlers_parallel_issue1146) { + ASSERT_TRUE(et_handles_exact(et_laravel_class_handlers, ET_LARAVEL_CLASS_FILES, + et_laravel_class_handles, 1)); + PASS(); +} + +/* Without composer.json there is no PSR-4 map: a class is placed only where + * its namespace mirrors the folders (App\Http\Controllers -> app/Http/ + * Controllers). The package class, whose folder does not mirror its + * namespace, then gets no handler rather than a guessed one. */ +TEST(handles_laravel_class_handlers_no_composer_issue1146) { + static const EtHandles want[] = { + {"Controllers.UserController.UserController.show", "/users/{id}"}, + {"GetCurrentUserController.__invoke", "/me"}, + {"PostController.show", "/posts/{id}"}, + {NULL, NULL}}; + ASSERT_TRUE( + et_handles_exact(et_laravel_class_handlers + 1, ET_LARAVEL_CLASS_FILES - 1, want, 0)); + PASS(); +} + /* Rails (Ruby) — ActionDispatch router. The handler MUST be passed as a bare * identifier (not the idiomatic `to: 'list_items'` string, which extract_handler_arg * cannot capture). mapper.get resolves by name to the Mapper#get method whose QN @@ -2138,6 +2301,9 @@ SUITE(edge_types_probe) { RUN_TEST(routes_laravel_routeserviceprovider_control_issue1146); RUN_TEST(routes_laravel_withrouting_nonliteral_prefix_issue1146); RUN_TEST(routes_laravel_withrouting_parallel_subdir_issue1146); + RUN_TEST(handles_laravel_class_handlers_issue1146); + RUN_TEST(handles_laravel_class_handlers_parallel_issue1146); + RUN_TEST(handles_laravel_class_handlers_no_composer_issue1146); RUN_TEST(handles_rails_ruby); RUN_TEST(handles_actix_rust); From 42118ad1427ab2a59a502ec72028d803913c913b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 21:11:49 +0200 Subject: [PATCH 25/34] fix(routes): resolve PSR-4 handler classes through the class-file resolver Why: since 830f0ccf (#2509) the package map stores a composer.json PSR-4 prefix ("Acme\Blog\") as its repo-relative directory ("packages/blog/src"), not as a dotted QN root. The route-handler resolver still formatted that value as a QN root and built "packages/blog/src.Http.PostsController. PostsController.index", which names no node. The PSR-4 step therefore resolved nothing. Handlers whose folders mirror their namespace (App\Http\Controllers -> app/Http/Controllers) still got their edge through the namespace/folder fallback, but a package class under a root whose folder does not mirror its namespace got no HANDLES edge. That is the case of packages/blog/src in the test fixture and of krayin's Webkul\Admin\ -> packages/Webkul/Admin/src. Change: resolve the handler class the way the import pass resolves `use Ns\Class;` for PHP. - pass_pkgmap.c: psr4_file_node and resolve_php_psr4_class take the graph buffer instead of the pipeline ctx, since they only read ctx->gbuf. The new cbm_pipeline_psr4_member_qn runs resolve_php_psr4_class for the class FQN, takes the Class/Interface/Trait/Enum node of that name defined in the resolved file, and returns the QN of . when that node exists. - registry.c: psr4_member_qn delegates to it and no longer builds a QN from the package-map value. cbm_registry_resolve_handler takes the graph buffer in place of the package map. The namespace/folder fallback is unchanged. - pass_calls.c and pass_parallel.c pass the graph buffer they already use for the handler lookup: ctx->gbuf, and main_gbuf, which is read-only during the resolve phase. Proof: - handles_laravel_class_handlers_issue1146 and its _parallel twin (edge_types_probe): RED 3/3 before the change ("missing PostsController.index -> /blog", expected=4 actual=3; suite 75 passed, 2 failed). GREEN 3/3 after (77 passed). RED again with the production change reverted. GREEN again once it was restored. - Real input: no Laravel app with a non-mirroring PSR-4 root is available locally. koel/koel v9.11.1 (App\ -> app/) was indexed with the binary before and after, each under a private runtime and cache. The results are identical: 16,564 nodes, 60,177 edges, 15 Routes and 1 HANDLES (/demo/new-session -> NewSessionController.__invoke), with the same handler rows. - Suites: edge_types_probe 77, edge_imports 68, pipeline 317 and route_canon 11 all pass (473). make lint-ci passes. Refs #1146 Signed-off-by: Martin Vogel --- src/pipeline/pass_calls.c | 2 +- src/pipeline/pass_parallel.c | 4 +-- src/pipeline/pass_pkgmap.c | 52 +++++++++++++++++++++++++++----- src/pipeline/pipeline_internal.h | 20 +++++++++--- src/pipeline/registry.c | 49 +++++++++--------------------- 5 files changed, 77 insertions(+), 50 deletions(-) diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 53f57025a..a40f8947c 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -406,7 +406,7 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { cbm_resolution_t hres = cbm_registry_resolve_handler(ctx->registry, call->second_arg_name, module_qn, imp_keys, - imp_vals, imp_count, cbm_pipeline_get_pkgmap()); + imp_vals, imp_count, ctx->gbuf); if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *handler = cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); if (handler != NULL) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index dfaea7161..7514008ce 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2295,8 +2295,8 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou esc_rp); cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); if (handler_ref && handler_ref[0] != '\0') { - cbm_resolution_t hres = cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, - iv, ic, cbm_pipeline_get_pkgmap()); + cbm_resolution_t hres = + cbm_registry_resolve_handler(registry, handler_ref, module_qn, ik, iv, ic, main_gbuf); if (hres.qualified_name && hres.qualified_name[0] != '\0') { const cbm_gbuf_node_t *h = cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); if (h) { diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index 9c4ace14b..71d3d3bec 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -2060,10 +2060,10 @@ typedef enum { /* The File node at exactly `rel_path`, or NULL. Looked up through the name * index (File nodes are named by basename) and matched on the full path, so * a same-named file in another directory never qualifies. */ -static const cbm_gbuf_node_t *psr4_file_node(const cbm_pipeline_ctx_t *ctx, const char *rel_path) { +static const cbm_gbuf_node_t *psr4_file_node(const cbm_gbuf_t *gbuf, const char *rel_path) { const cbm_gbuf_node_t **hits = NULL; int hit_count = 0; - if (cbm_gbuf_find_by_name(ctx->gbuf, path_leaf(rel_path), &hits, &hit_count) != 0 || !hits) { + if (cbm_gbuf_find_by_name(gbuf, path_leaf(rel_path), &hits, &hit_count) != 0 || !hits) { return NULL; } for (int i = 0; i < hit_count; i++) { @@ -2084,9 +2084,8 @@ static const cbm_gbuf_node_t *psr4_file_node(const cbm_pipeline_ctx_t *ctx, cons * to the namespace bucket, which binds it to whichever file of the namespace * came first. `use function` / `use const` name namespace members, not class * files, so they are not applicable. */ -static psr4_outcome_t resolve_php_psr4_class(const cbm_pipeline_ctx_t *ctx, - const char *source_file_qn, const CBMImport *imp, - const cbm_gbuf_node_t **out) { +static psr4_outcome_t resolve_php_psr4_class(const cbm_gbuf_t *gbuf, const char *source_file_qn, + const CBMImport *imp, const cbm_gbuf_node_t **out) { *out = NULL; CBMHashTable *pkgmap = cbm_pipeline_get_pkgmap(); if (!pkgmap || imp->kind != CBM_IMPORT_KIND_DEFAULT) { @@ -2124,7 +2123,7 @@ static psr4_outcome_t resolve_php_psr4_class(const cbm_pipeline_ctx_t *ctx, *c = '/'; } } - const cbm_gbuf_node_t *file = psr4_file_node(ctx, rel); + const cbm_gbuf_node_t *file = psr4_file_node(gbuf, rel); if (file && (!source_file_qn || !file->qualified_name || strcmp(file->qualified_name, source_file_qn) != 0)) { *out = file; @@ -2134,6 +2133,45 @@ static psr4_outcome_t resolve_php_psr4_class(const cbm_pipeline_ctx_t *ctx, return covered ? PSR4_UNRESOLVED : PSR4_NOT_APPLICABLE; } +static bool psr4_class_label(const char *label) { + return label && (strcmp(label, "Class") == 0 || strcmp(label, "Interface") == 0 || + strcmp(label, "Trait") == 0 || strcmp(label, "Enum") == 0); +} + +const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, + const char *member) { + if (!gbuf || !class_fqn || !member || !member[0]) { + return NULL; + } + const CBMImport imp = {.module_path = class_fqn, .kind = CBM_IMPORT_KIND_DEFAULT}; + const cbm_gbuf_node_t *file = NULL; + if (resolve_php_psr4_class(gbuf, NULL, &imp, &file) != PSR4_RESOLVED || !file->file_path) { + return NULL; + } + const char *cls = strrchr(class_fqn, '\\'); + cls = cls ? cls + SKIP_ONE : class_fqn; + const cbm_gbuf_node_t **hits = NULL; + int hit_count = 0; + if (cbm_gbuf_find_by_name(gbuf, cls, &hits, &hit_count) != 0 || !hits) { + return NULL; + } + for (int i = 0; i < hit_count; i++) { + const cbm_gbuf_node_t *c = hits[i]; + if (!c || !psr4_class_label(c->label) || !c->qualified_name || !c->file_path || + strcmp(c->file_path, file->file_path) != 0) { + continue; + } + char qn[CBM_SZ_1K]; + int w = snprintf(qn, sizeof(qn), "%s.%s", c->qualified_name, member); + if (w <= 0 || (size_t)w >= sizeof(qn)) { + return NULL; + } + const cbm_gbuf_node_t *m = cbm_gbuf_find_by_qn(gbuf, qn); + return m ? m->qualified_name : NULL; + } + return NULL; +} + const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, const char *source_rel, const char *source_file_qn, @@ -2154,7 +2192,7 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t /* PHP class imports covered by a composer psr-4 prefix name exactly one * file; when it is absent the import stays unresolved (#1186). */ const cbm_gbuf_node_t *psr4_target = NULL; - switch (resolve_php_psr4_class(ctx, source_file_qn, imp, &psr4_target)) { + switch (resolve_php_psr4_class(ctx->gbuf, source_file_qn, imp, &psr4_target)) { case PSR4_RESOLVED: return psr4_target; case PSR4_UNRESOLVED: diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 451776807..650f8a08d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -213,14 +213,24 @@ void cbm_pipeline_set_pkgmap(CBMHashTable *map); * Laravel `[Class::class, 'method']` and invokable `Class::class` handlers * (#1146), with the class already qualified through the file's `use` imports * and namespace — is placed on a method QN only by what places the class file: - * a composer.json PSR-4 root in psr4_roots (the package map), else the one - * candidate whose directories end with the class's namespace. Never by short - * name alone, so a vendor controller that is not in the repo gets no handler. - * A reference without "::" resolves exactly as cbm_registry_resolve does. */ + * the composer.json PSR-4 class file in gbuf (cbm_pipeline_psr4_member_qn), + * else the one candidate whose directories end with the class's namespace. + * Never by short name alone, so a vendor controller that is not in the repo + * gets no handler. A reference without "::" resolves exactly as + * cbm_registry_resolve does. */ cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count, - const CBMHashTable *psr4_roots); + const cbm_gbuf_t *gbuf); + +/* The QN of `member` on the PHP class `class_fqn` ("Ns\\Class", NUL- + * terminated), placed the way composer's PSR-4 autoloader places the class: + * the class file the import resolver picks for `use Ns\\Class;` (#1186), the + * class that file defines, then that class's member. NULL when no PSR-4 + * prefix covers the class, its file is absent, or the class has no such + * member. The returned string is owned by gbuf. Read-only on gbuf. */ +const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, + const char *member); /* Unified module resolver: relative → pkgmap → fqn_module fallback. * Handles bare specifiers via pkgmap lookup with prefix matching. diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 4159c19f4..3e564f816 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -1502,42 +1502,21 @@ static int qn_class_member_tail(const char *qn, const char *cls, size_t cls_len, return (int)(c - SKIP_ONE - qn); } -/* PSR-4: the longest composer.json autoload root "Prefix\\" that prefixes the - * FQN maps the remaining namespace onto directories below the root's folder - * and the class onto .php — the file Composer's autoloader loads. - * Returns the registered method QN, or NULL (no root prefixes the FQN, or the - * autoloaded file holds no such method). */ -static const char *psr4_member_qn(const cbm_registry_t *r, const CBMHashTable *psr4_roots, - const char *fqn, size_t fqn_len, const char *cls, size_t cls_len, +/* PSR-4: the class file Composer's autoloader loads for the FQN, the class it + * defines, and that class's member — through the PSR-4 import resolver + * (cbm_pipeline_psr4_member_qn), so a root whose folder does not mirror its + * namespace (Acme\\Blog\\ -> packages/blog/src) places the class as well. + * Returns the member QN, or NULL (no root covers the FQN, its class file is + * absent, or the class holds no such member). */ +static const char *psr4_member_qn(const cbm_gbuf_t *gbuf, const char *fqn, size_t fqn_len, const char *member) { - if (!psr4_roots || fqn_len >= CBM_SZ_512) { + char cls_fqn[CBM_SZ_512]; + if (!gbuf || fqn_len >= sizeof(cls_fqn)) { return NULL; } - char key[CBM_SZ_512]; - for (size_t cut = fqn_len; cut > 0; cut--) { - if (fqn[cut - SKIP_ONE] != '\\') { - continue; - } - memcpy(key, fqn, cut); /* "App\\" / "App\\Http\\" — keeps the trailing '\\' */ - key[cut] = '\0'; - const char *root_qn = (const char *)cbm_ht_get(psr4_roots, key); - if (!root_qn) { - continue; - } - char qn[CBM_SZ_1K]; - int n = snprintf(qn, sizeof(qn), "%s.%.*s.%.*s.%s", root_qn, (int)(fqn_len - cut), - fqn + cut, (int)cls_len, cls, member); - if (n <= 0 || (size_t)n >= sizeof(qn)) { - return NULL; - } - for (char *p = qn + strlen(root_qn); *p; p++) { - if (*p == '\\') { - *p = '.'; - } - } - return cbm_ht_get_key(r->exact, qn); - } - return NULL; + memcpy(cls_fqn, fqn, fqn_len); + cls_fqn[fqn_len] = '\0'; + return cbm_pipeline_psr4_member_qn(gbuf, cls_fqn, member); } /* Do the directory segments of qn (between the project root segment and the @@ -1600,7 +1579,7 @@ static const char *aligned_member_qn(const cbm_registry_t *r, const char *fqn, s cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count, - const CBMHashTable *psr4_roots) { + const cbm_gbuf_t *gbuf) { const char *sep = handler_ref ? strstr(handler_ref, "::") : NULL; if (!r || !sep) { return cbm_registry_resolve(r, handler_ref, module_qn, import_map_keys, import_map_vals, @@ -1618,7 +1597,7 @@ cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const cha if (cls_len == 0 || !member[0]) { return empty_result(); } - const char *qn = psr4_member_qn(r, psr4_roots, handler_ref, fqn_len, cls, cls_len, member); + const char *qn = psr4_member_qn(gbuf, handler_ref, fqn_len, member); if (qn) { return (cbm_resolution_t){qn, "php_psr4", CONF_PSR4_MEMBER, REG_RESOLVED}; } From 2633334c59a8cdd27378d1d42dce20e88816be93 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 22:06:47 +0200 Subject: [PATCH 26/34] ci: analyse and score only the newest main commit CodeQL SAST keyed its concurrency group by commit SHA on push events, so every merge to main got its own group and nothing was ever cancelled; OpenSSF Scorecard had no group at all. On 2026-10-04 a day of 36 merges queued one CodeQL analysis (40-100 minutes each) and one Scorecard run per merge, ahead of every pull request run: pull request CI waited hours for a runner, and a main analysis waited 4 hours. CodeQL now groups by event and by pull request number or ref, so pull requests keep one analysis each as before, and a newer push to main cancels the analysis of an older main commit. Scorecard groups by ref with the same cancellation. Only the newest main commit matters for both: code scanning tracks the default branch's latest analysis. DCO keeps its per-commit group on purpose: its push run on main is the check that catches a merge rewriting sign-offs, so a superseded push must still be checked, and a DCO job takes seconds. Signed-off-by: Martin Vogel --- .github/workflows/codeql.yml | 6 +++++- .github/workflows/scorecard.yml | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2430c6712..621d407fc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,8 +9,12 @@ on: permissions: contents: read +# One analysis per pull request and one per pushed branch. A newer push to main +# cancels the analysis of an older main commit; only the newest commit's +# results matter for main, and a merge-heavy day otherwise queues one 40-100 +# minute analysis per merge ahead of every pull request run. concurrency: - group: codeql-${{ github.ref }}-${{ github.event.pull_request.number || github.sha }} + group: codeql-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index c2acbd8ac..fb0a52bbc 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -8,6 +8,12 @@ on: permissions: read-all +# Only the newest main commit is scored: a newer push (or the weekly run) +# cancels a pending or running score of an older commit. +concurrency: + group: scorecard-${{ github.ref }} + cancel-in-progress: true + jobs: scorecard: runs-on: ubuntu-latest From f8d8fcdefec1cb5d28a064b98270a7fe3a5c45ac Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 4 Oct 2026 22:27:29 +0200 Subject: [PATCH 27/34] test: keep ASan stack-use-after-return off outside the diag lane gcc's libasan turns detect_stack_use_after_return ON by default; clang's runtime leaves it off. On aarch64, gcc 13/14 free small fake frames without clearing their in-use flag, so once 8,192 frames of a size class have been used, every instrumented call scans the flag array (~8 us instead of ~6 ns) and the check silently stops detecting anything. CI's ubuntu-24.04-arm gcc leg runs the extraction suite in 688 s against a 900 s cap (16 s on amd64), and the deep extraction tests of #2478 push it over the cap. The lanes already document the intent: test-diag in _test.yml and `make -f Makefile.cbm diag` set detect_stack_use_after_return=1 explicitly, as one of the ASan checks that are off by default in every other lane. scripts/test.sh now appends detect_stack_use_after_return=0 to ASAN_OPTIONS (keeping any options already set, ':'-separated) unless ASAN_OPTIONS already mentions the flag, and exports it, so test-diag's explicit =1 wins. Lanes built with clang already ran with it off and are unchanged. The venue-parity contract (Step 0j) gains a layer that drives the real scripts/test.sh with a no-op make and a probe runner and asserts three cases: unset, other options kept with the flag appended, explicit =1 kept. Measured in the local Linux arm64 container (Colima, gcc 13.3, ASan + UBSan), wall time of `scripts/test.sh --suites extraction`: origin/main 268a9d88 without this change 628.6 s origin/main 268a9d88 with this change 5.1 s #2478 head 818ef0ec with this change 9.7 s Signed-off-by: Martin Vogel --- scripts/test.sh | 12 ++++++ tests/test_venue_parity_contract.sh | 65 ++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/scripts/test.sh b/scripts/test.sh index 1741f8a1d..54962bf04 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -179,6 +179,18 @@ if [ "$SANITIZE_GIVEN" -eq 0 ] && [ "${MSYSTEM:-}" = "CLANGARM64" ]; then MAKE_ARGS+=("SANITIZE=-fsanitize=undefined -fsanitize-trap=undefined -fstack-protector-strong -fno-omit-frame-pointer") fi +# ASan stack-use-after-return runs in the test-diag lane only (_test.yml and +# `make -f Makefile.cbm diag` set detect_stack_use_after_return=1). gcc's +# libasan turns it ON by default, and on aarch64 gcc 13/14 never clear a freed +# fake frame's in-use flag: past 8,192 frames per size class every instrumented +# call scans that flag array and the check stops detecting anything (gcc arm64 +# ran `extraction` in 688 s, amd64 in 16 s). Off here; an explicit setting wins. +case "${ASAN_OPTIONS:-}" in + *detect_stack_use_after_return*) ;; + *) ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}detect_stack_use_after_return=0" ;; +esac +export ASAN_OPTIONS + EXPECTED_SANITIZED=1 if [ "$SANITIZE_GIVEN" -eq 1 ]; then case "$SANITIZE_VALUE" in diff --git a/tests/test_venue_parity_contract.sh b/tests/test_venue_parity_contract.sh index 81e431a1e..cef82b33e 100755 --- a/tests/test_venue_parity_contract.sh +++ b/tests/test_venue_parity_contract.sh @@ -24,10 +24,13 @@ # 5. Interface probes — every entry must answer --help (exit 0, "Usage:") # and reject unknown flags with exit 2 + "Please consult --help." so an # agent can always discover exactly what a run will do. +# 6. Sanitizer runtime defaults — scripts/test.sh, not each venue, sets the +# ASan options every lane runs with (stack-use-after-return stays off +# unless a lane asks for it explicitly, as test-diag does). # # Usage: tests/test_venue_parity_contract.sh [repo-root] (root override so # the suite can prove the contract FAILS on a violation, not just passes; -# layer 5 runs only against the real repo root). +# layers 5 and 6 run only against the real repo root). set -euo pipefail @@ -455,3 +458,63 @@ scripts/memwaste.sh fi echo "interface probes OK ($(printf '%s' "$HELP_ENTRIES" | grep -c .) --help entries, $(printf '%s' "$STRICT_ENTRIES" | grep -c .) strict-flag entries)" fi + +# ── Layer 6: sanitizer runtime defaults (real repo root only). Drives the REAL +# scripts/test.sh in --suites mode with a no-op `make` first on PATH and a +# probe standing in for the test-runner: the probe answers the build-config +# handshake, then records the ASAN_OPTIONS it inherited. Every case sets or +# unsets ASAN_OPTIONS itself, because this contract runs INSIDE test.sh, whose +# own export would otherwise leak in. +if [ -d "$ROOT/scripts" ] && [ -f "$ROOT/scripts/test.sh" ]; then + asan_failures=0 + asan_tmp="$(mktemp -d)" + trap 'rm -rf "$asan_tmp"' EXIT + mkdir -p "$asan_tmp/bin" "$asan_tmp/build" + printf '#!/usr/bin/env bash\nexit 0\n' > "$asan_tmp/bin/make" + cat > "$asan_tmp/build/test-runner" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = "--build-config" ]; then + printf 'sanitized=1 test_seams=1\n' + exit 0 +fi +printf '%s' "${ASAN_OPTIONS-}" > "$CBM_ASAN_PROBE_OUT" +EOF + chmod +x "$asan_tmp/bin/make" "$asan_tmp/build/test-runner" + + # asan_case LABEL GIVEN WANT: GIVEN is the caller's ASAN_OPTIONS ("" + # for none), WANT the exact value the runner must receive. + asan_case() { + local label="$1" given="$2" want="$3" seen rc + rm -f "$asan_tmp/seen" + ( + if [ "$given" = "" ]; then + unset ASAN_OPTIONS + else + export ASAN_OPTIONS="$given" + fi + cd "$ROOT" && PATH="$asan_tmp/bin:$PATH" CBM_NO_CCACHE=1 \ + CBM_ASAN_PROBE_OUT="$asan_tmp/seen" \ + bash scripts/test.sh --suites probe "BUILD_DIR=$asan_tmp/build" + ) >"$asan_tmp/log" 2>&1 && rc=0 || rc=$? + seen="$(cat "$asan_tmp/seen" 2>/dev/null || printf '')" + if [ "$rc" -ne 0 ] || [ "$seen" != "$want" ]; then + echo "SANITIZER DEFAULTS: $label: runner got ASAN_OPTIONS=[$seen], want [$want] (test.sh rc=$rc)" >&2 + sed 's/^/ /' "$asan_tmp/log" >&2 + asan_failures=1 + fi + } + + asan_case "unset" "" \ + "detect_stack_use_after_return=0" + asan_case "other options kept" "detect_leaks=1:halt_on_error=1" \ + "detect_leaks=1:halt_on_error=1:detect_stack_use_after_return=0" + asan_case "explicit setting wins (test-diag)" \ + "detect_stack_use_after_return=1:strict_string_checks=1:detect_stack_use_after_scope=1" \ + "detect_stack_use_after_return=1:strict_string_checks=1:detect_stack_use_after_scope=1" + + if [ "$asan_failures" -ne 0 ]; then + echo "VENUE PARITY CONTRACT VIOLATED — sanitizer runtime defaults failed (layer 6)" >&2 + exit 1 + fi + echo "sanitizer runtime defaults OK (3 ASAN_OPTIONS cases through scripts/test.sh)" +fi From 70e73442ed3ced71619d561b901af92729ede595 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 01:37:11 +0200 Subject: [PATCH 28/34] fix(routes): leave a PSR-4-covered handler without a class file unresolved Why: when a composer.json PSR-4 prefix covers a handler class but no class file exists for it, cbm_registry_resolve_handler fell through to the namespace/folder fallback and bound the route to a same-named class in any folder that mirrors the namespace, a class composer never loads. Main's import resolver stops in that case: resolve_php_psr4_class leaves the `use` import unresolved (#1186). The handler now does the same, so no guessed HANDLES edge is minted. Classes that no PSR-4 prefix covers keep the fallback unchanged. Change: - pipeline_internal.h: cbm_pipeline_psr4_member_qn returns a tri-state cbm_psr4_member_t (NOT_COVERED, RESOLVED, UNRESOLVED) and hands out the member QN through an out parameter, set only when RESOLVED. UNRESOLVED means a prefix covers the class but no class file holds the class with that member: the file is absent, or it lacks the class or the member. - pass_pkgmap.c: maps the outcome of resolve_php_psr4_class onto it; the class/member lookup in the resolved file moves into psr4_member_in_file. - registry.c: cbm_registry_resolve_handler returns unresolved on UNRESOLVED and runs the folder fallback only on NOT_COVERED. Proof: - New handles_laravel_psr4_absent_class_issue1146 and its _parallel twin (edge_types_probe) extend the issue-1146 fixture with a route to \Acme\Blog\Http\MissingController::index (covered by Acme\Blog\ -> packages/blog/src/, no class file) and one to PostsController::archive (class file exists, no archive method), plus namespace-mirroring decoys in legacy/Acme/Blog/Http that hold both members. The test asserts exactly the 4 HANDLES of the fixture and that both Route nodes exist. RED before the change on both paths with the 2 decoy edges (legacy...MissingController.index -> /blog/missing, legacy...PostsController.archive -> /blog/archive; expected=4 actual=6; suite 77 passed, 2 failed). GREEN after (79 passed). RED again with the production change reverted (the same 2 edges), GREEN once it was restored. - Real input: koel/koel v9.11.1 (App\ -> app/) indexed with the new binary under a private HOME, cache and runtime: 16,564 nodes, 60,177 edges, 15 Routes and 1 HANDLES (/demo/new-session -> NewSessionController.__invoke), identical to the numbers recorded for 42118ad1. - Suites: edge_types_probe 79, edge_imports 68, pipeline 317 and route_canon 11 all pass (475). make lint-ci passes. Refs #1146 Signed-off-by: Martin Vogel --- src/pipeline/pass_pkgmap.c | 41 ++++++++++++------ src/pipeline/pipeline_internal.h | 40 ++++++++++++------ src/pipeline/registry.c | 23 ++++++---- tests/test_edge_types_probe.c | 72 +++++++++++++++++++++++++++++++- 4 files changed, 141 insertions(+), 35 deletions(-) diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index 71d3d3bec..c82cdf645 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -2138,18 +2138,10 @@ static bool psr4_class_label(const char *label) { strcmp(label, "Trait") == 0 || strcmp(label, "Enum") == 0); } -const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, - const char *member) { - if (!gbuf || !class_fqn || !member || !member[0]) { - return NULL; - } - const CBMImport imp = {.module_path = class_fqn, .kind = CBM_IMPORT_KIND_DEFAULT}; - const cbm_gbuf_node_t *file = NULL; - if (resolve_php_psr4_class(gbuf, NULL, &imp, &file) != PSR4_RESOLVED || !file->file_path) { - return NULL; - } - const char *cls = strrchr(class_fqn, '\\'); - cls = cls ? cls + SKIP_ONE : class_fqn; +/* The QN of `member` on the class `cls` defined in `file_path`, or NULL when + * that file defines no such class or the class has no such member. */ +static const char *psr4_member_in_file(const cbm_gbuf_t *gbuf, const char *file_path, + const char *cls, const char *member) { const cbm_gbuf_node_t **hits = NULL; int hit_count = 0; if (cbm_gbuf_find_by_name(gbuf, cls, &hits, &hit_count) != 0 || !hits) { @@ -2158,7 +2150,7 @@ const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *clas for (int i = 0; i < hit_count; i++) { const cbm_gbuf_node_t *c = hits[i]; if (!c || !psr4_class_label(c->label) || !c->qualified_name || !c->file_path || - strcmp(c->file_path, file->file_path) != 0) { + strcmp(c->file_path, file_path) != 0) { continue; } char qn[CBM_SZ_1K]; @@ -2172,6 +2164,29 @@ const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *clas return NULL; } +cbm_psr4_member_t cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, + const char *member, const char **out_qn) { + *out_qn = NULL; + if (!gbuf || !class_fqn || !member || !member[0]) { + return CBM_PSR4_MEMBER_NOT_COVERED; + } + const CBMImport imp = {.module_path = class_fqn, .kind = CBM_IMPORT_KIND_DEFAULT}; + const cbm_gbuf_node_t *file = NULL; + psr4_outcome_t placed = resolve_php_psr4_class(gbuf, NULL, &imp, &file); + if (placed == PSR4_NOT_APPLICABLE) { + return CBM_PSR4_MEMBER_NOT_COVERED; + } + /* Covered from here on: composer would load this file or none, so a + * missing file, class or member leaves the member unresolved. */ + if (placed != PSR4_RESOLVED || !file->file_path) { + return CBM_PSR4_MEMBER_UNRESOLVED; + } + const char *cls = strrchr(class_fqn, '\\'); + cls = cls ? cls + SKIP_ONE : class_fqn; + *out_qn = psr4_member_in_file(gbuf, file->file_path, cls, member); + return *out_qn ? CBM_PSR4_MEMBER_RESOLVED : CBM_PSR4_MEMBER_UNRESOLVED; +} + const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, const char *source_rel, const char *source_file_qn, diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 650f8a08d..fb3ed1253 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -213,24 +213,38 @@ void cbm_pipeline_set_pkgmap(CBMHashTable *map); * Laravel `[Class::class, 'method']` and invokable `Class::class` handlers * (#1146), with the class already qualified through the file's `use` imports * and namespace — is placed on a method QN only by what places the class file: - * the composer.json PSR-4 class file in gbuf (cbm_pipeline_psr4_member_qn), - * else the one candidate whose directories end with the class's namespace. - * Never by short name alone, so a vendor controller that is not in the repo - * gets no handler. A reference without "::" resolves exactly as - * cbm_registry_resolve does. */ + * the composer.json PSR-4 class file in gbuf (cbm_pipeline_psr4_member_qn). + * When a PSR-4 prefix covers the class, that answer is final: a covered class + * whose file or member is absent stays unresolved, as its `use` import does + * (#1186). Only a class no prefix covers falls back to the one candidate whose + * directories end with the class's namespace. Never by short name alone, so a + * vendor controller that is not in the repo gets no handler. A reference + * without "::" resolves exactly as cbm_registry_resolve does. */ cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const char *handler_ref, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count, const cbm_gbuf_t *gbuf); -/* The QN of `member` on the PHP class `class_fqn` ("Ns\\Class", NUL- - * terminated), placed the way composer's PSR-4 autoloader places the class: - * the class file the import resolver picks for `use Ns\\Class;` (#1186), the - * class that file defines, then that class's member. NULL when no PSR-4 - * prefix covers the class, its file is absent, or the class has no such - * member. The returned string is owned by gbuf. Read-only on gbuf. */ -const char *cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, - const char *member); +/* Where composer's PSR-4 autoloader places a PHP class member. Tri-state, so + * "no prefix covers the class" (another strategy may answer) is never + * confused with "a prefix covers it but places nothing" (final). */ +typedef enum { + CBM_PSR4_MEMBER_NOT_COVERED = 0, /* no psr-4 prefix covers the class */ + CBM_PSR4_MEMBER_RESOLVED, /* *out_qn is the member's QN */ + CBM_PSR4_MEMBER_UNRESOLVED, /* covered, but no class file holds the + * class with this member: leave it + * unresolved, never guess elsewhere */ +} cbm_psr4_member_t; + +/* Place `member` on the PHP class `class_fqn` ("Ns\\Class", NUL-terminated) + * the way composer's PSR-4 autoloader places the class: the class file the + * import resolver picks for `use Ns\\Class;` (#1186), the class that file + * defines, then that class's member. *out_qn is set only on + * CBM_PSR4_MEMBER_RESOLVED (else NULL) and is owned by gbuf. Missing + * arguments or no package map give CBM_PSR4_MEMBER_NOT_COVERED. Read-only on + * gbuf. */ +cbm_psr4_member_t cbm_pipeline_psr4_member_qn(const cbm_gbuf_t *gbuf, const char *class_fqn, + const char *member, const char **out_qn); /* Unified module resolver: relative → pkgmap → fqn_module fallback. * Handles bare specifiers via pkgmap lookup with prefix matching. diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 3e564f816..1e4e0819e 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -1506,17 +1506,17 @@ static int qn_class_member_tail(const char *qn, const char *cls, size_t cls_len, * defines, and that class's member — through the PSR-4 import resolver * (cbm_pipeline_psr4_member_qn), so a root whose folder does not mirror its * namespace (Acme\\Blog\\ -> packages/blog/src) places the class as well. - * Returns the member QN, or NULL (no root covers the FQN, its class file is - * absent, or the class holds no such member). */ -static const char *psr4_member_qn(const cbm_gbuf_t *gbuf, const char *fqn, size_t fqn_len, - const char *member) { + * Returns its tri-state outcome; *out_qn is the member QN when RESOLVED. */ +static cbm_psr4_member_t psr4_member_qn(const cbm_gbuf_t *gbuf, const char *fqn, size_t fqn_len, + const char *member, const char **out_qn) { char cls_fqn[CBM_SZ_512]; + *out_qn = NULL; if (!gbuf || fqn_len >= sizeof(cls_fqn)) { - return NULL; + return CBM_PSR4_MEMBER_NOT_COVERED; } memcpy(cls_fqn, fqn, fqn_len); cls_fqn[fqn_len] = '\0'; - return cbm_pipeline_psr4_member_qn(gbuf, cls_fqn, member); + return cbm_pipeline_psr4_member_qn(gbuf, cls_fqn, member, out_qn); } /* Do the directory segments of qn (between the project root segment and the @@ -1597,9 +1597,16 @@ cbm_resolution_t cbm_registry_resolve_handler(const cbm_registry_t *r, const cha if (cls_len == 0 || !member[0]) { return empty_result(); } - const char *qn = psr4_member_qn(gbuf, handler_ref, fqn_len, member); - if (qn) { + const char *qn = NULL; + switch (psr4_member_qn(gbuf, handler_ref, fqn_len, member, &qn)) { + case CBM_PSR4_MEMBER_RESOLVED: return (cbm_resolution_t){qn, "php_psr4", CONF_PSR4_MEMBER, REG_RESOLVED}; + case CBM_PSR4_MEMBER_UNRESOLVED: + /* A covered class lives where PSR-4 says or nowhere; a namespace- + * mirroring folder elsewhere would be a guessed edge (#1186). */ + return empty_result(); + case CBM_PSR4_MEMBER_NOT_COVERED: + break; } qn = aligned_member_qn(r, handler_ref, ns_len, cls, cls_len, member); if (qn) { diff --git a/tests/test_edge_types_probe.c b/tests/test_edge_types_probe.c index 999019c45..6dfa94e1c 100644 --- a/tests/test_edge_types_probe.c +++ b/tests/test_edge_types_probe.c @@ -939,7 +939,11 @@ static int et_qn_has_tail(const char *qn, const char *tail) { return ql >= tl && strcmp(qn + ql - tl, tail) == 0 && (ql == tl || qn[ql - tl - 1] == '.'); } -static int et_handles_exact(const EtFile *files, int nfiles, const EtHandles *want, int parallel) { +/* route_qns (optional, NULL-terminated): Route qualified_names that must + * exist, so a test asserting that a route gets NO handler cannot pass merely + * because the route itself was never extracted. */ +static int et_handles_exact_routes(const EtFile *files, int nfiles, const EtHandles *want, + int parallel, const char *const *route_qns) { EtProj lp; cbm_store_t *store = parallel ? et_index_parallel(&lp, files, nfiles) : et_index_files(&lp, files, nfiles); @@ -983,6 +987,16 @@ static int et_handles_exact(const EtFile *files, int nfiles, const EtHandles *wa want[wi].route); } } + for (int ri = 0; route_qns && route_qns[ri]; ri++) { + cbm_node_t route = {0}; + if (!store || + cbm_store_find_node_by_qn(store, lp.project, route_qns[ri], &route) != CBM_STORE_OK) { + ok = 0; + fprintf(stderr, " [ET-HANDLES] missing Route %s\n", route_qns[ri]); + } else { + cbm_node_free_fields(&route); + } + } if (!ok) { fprintf(stderr, " [ET-HANDLES] FAIL (%s path) expected=%d actual=%d\n", parallel ? "parallel" : "sequential", wanted, n); @@ -992,6 +1006,10 @@ static int et_handles_exact(const EtFile *files, int nfiles, const EtHandles *wa return ok; } +static int et_handles_exact(const EtFile *files, int nfiles, const EtHandles *want, int parallel) { + return et_handles_exact_routes(files, nfiles, want, parallel, NULL); +} + /* #1146: Laravel's two class-based handler forms — * Route::get('/x', [UserController::class, 'show']) (controller method) * Route::get('/x', GetCurrentUserController::class) (invokable: __invoke) @@ -1085,6 +1103,56 @@ TEST(handles_laravel_class_handlers_no_composer_issue1146) { PASS(); } +/* A PSR-4 prefix that covers a handler class decides where the class lives, + * as it does for the class's `use` import (#1186): when composer would load + * no file holding the member, the route stays without a handler. Both routes + * below are covered by Acme\Blog\ -> packages/blog/src/: + * /blog/missing MissingController has no class file there at all; + * /blog/archive PostsController's class file exists but has no archive(). + * legacy/Acme/Blog/Http mirrors the namespace and holds both members, so the + * namespace/folder fallback would bind both routes to a class composer never + * loads. It must not run for a covered class. These files are added to the + * fixture above, whose four handlers must keep their edges. */ +static const EtFile et_laravel_psr4_absent_extra[] = { + {"legacy/Acme/Blog/Http/MissingController.php", + " true]; }\n}\n"}, + {"legacy/Acme/Blog/Http/PostsController.php", + " true]; }\n}\n"}, + {"routes/blog.php", + " Date: Wed, 7 Oct 2026 18:36:15 +0200 Subject: [PATCH 29/34] fix(extraction): parse C++ branch views under the terminated-last-line rule Merging main brought extract_cpp_branch_views (conditional variants), which built its CBMStringInput with the two-field initializer. With this branch's virtual_newline field that is a missing initializer, an error under GCC's -Wextra -Werror (the shadow job's product build). A branch view keeps the raw source's length and line structure, so it now goes through cbm_parse_source like the raw parse: the same virtual newline, and the same edit that removes it again. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index fb0d14099..210a0a067 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2267,8 +2267,6 @@ static void extract_cpp_branch_views(const CBMExtractCtx *raw, const TSLanguage return; } ts_parser_reset(parser); - CBMStringInput input = {view, (uint32_t)raw->source_len}; - TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL}; TSParseOptions opts = {0}; CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable if (timeout_micros > 0) { @@ -2278,7 +2276,9 @@ static void extract_cpp_branch_views(const CBMExtractCtx *raw, const TSLanguage opts.payload = &budget; opts.progress_callback = cbm_timeout_cb; } - TSTree *tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); + /* A view keeps the raw source's length and line structure, so it is + * parsed under the same terminated-last-line rule (#2078). */ + TSTree *tree = cbm_parse_source(parser, view, (uint32_t)raw->source_len, opts); if (!tree) { return; } From d64f7e5cad9046127e941a5d1265089288e6360c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 18:36:35 +0200 Subject: [PATCH 30/34] fix(extract): name an anonymous C typedef aggregate once Main (#2458) named the anonymous struct/union/enum of `typedef struct { ... } Name;` in extract_class_def from the typedef's declarator. This branch's extract_c_typedef already emits that aggregate as Name from the type_definition, so on the merged tree Name was emitted twice under one label: its members twice and a false variants entry. extract_c_typedef stays the one place that names it; the extract_class_def case is removed. Both typedef test sets pass on the result: #2458's extract_c_anonymous_typedef_aggregate_is_named_by_its_typedef and member-access tests, and this branch's C1 typedef/variant tests (extraction, pipeline, registry, graph_buffer, store_search, mcp: 1,336 passed, 4 skipped). Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 3bd1ecc51..d109995bd 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -7379,27 +7379,9 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec } break; } - case CBM_LANG_C: - case CBM_LANG_CPP: { // `typedef struct { … } Name;`: the aggregate is - // anonymous and the typedef's declarator names it. - // A pointer/array/function declarator names another - // type, not the aggregate, so only a plain - // type_identifier counts. - if (strcmp(kind, "struct_specifier") != 0 && strcmp(kind, "union_specifier") != 0 && - strcmp(kind, "enum_specifier") != 0) { - break; - } - TSNode parent = ts_node_parent(node); - if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "type_definition") != 0) { - break; - } - TSNode declarator = ts_node_child_by_field_name(parent, TS_FIELD("declarator")); - if (!ts_node_is_null(declarator) && - strcmp(ts_node_type(declarator), "type_identifier") == 0) { - name_node = declarator; - } - break; - } + /* C/C++ `typedef struct { … } Name;` is named by extract_c_typedef from + * the type_definition; naming the anonymous specifier here as well + * emitted Name twice (members twice, a false variant). */ default: break; } From c4d6453b6dd73f7ac57fb35d368b0209d921897b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 18:53:30 +0200 Subject: [PATCH 31/34] test(extract): an anonymous C typedef aggregate is one definition Regression test for the merge with #2458: with its extract_class_def case restored, typedef struct { int count; } Tally; yields two Class defs named Tally (count_defs_named == 2, RED at test_extraction.c:1129); with the case removed it is one Class, one count field, one Shade enum, and no variants list (extraction 432 passed). Signed-off-by: Martin Vogel --- tests/test_extraction.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 1390a25bc..c5057e7f7 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1116,6 +1116,29 @@ TEST(extract_c_anonymous_typedef_aggregate_is_named_by_its_typedef) { PASS(); } +/* The anonymous aggregate of a typedef is ONE definition. extract_c_typedef names + * it from the type_definition; a second naming from the anonymous specifier gave + * Tally two Class defs under one QN, every member twice, and a variants list. */ +TEST(extract_c_anonymous_typedef_aggregate_is_one_def) { + CBMFileResult *r = extract("typedef struct {\n" + " int count;\n" + "} Tally;\n" + "typedef enum { SHADE_RED, SHADE_GREEN } Shade;\n", + CBM_LANG_C, "t", "tally.h"); + ASSERT_NOT_NULL(r); + ASSERT_EQ(count_defs_named(r, "Class", "Tally"), 1); + ASSERT_EQ(count_defs_named(r, "Field", "count"), 1); + ASSERT_EQ(count_defs_named(r, "Enum", "Shade"), 1); + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->name && strcmp(d->name, "Tally") == 0) { + ASSERT_NULL(d->variants); + } + } + cbm_free_result(r); + PASS(); +} + /* A pointer-to-function member (`void (*open)(int);`, the shape of every * kernel ops table) is a field of its struct. It shares the * field_declaration + function_declarator shape with a C++ member FUNCTION @@ -10405,6 +10428,7 @@ SUITE(extraction) { RUN_TEST(c_function_return_type_plain_unchanged); RUN_TEST(c_struct); RUN_TEST(extract_c_anonymous_typedef_aggregate_is_named_by_its_typedef); + RUN_TEST(extract_c_anonymous_typedef_aggregate_is_one_def); RUN_TEST(extract_c_function_pointer_member_is_a_field); RUN_TEST(extract_c_member_declarators_name_and_type); RUN_TEST(cpp_class); From f9ce38658b083387350d8e1cad3977d65044f6a5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 19:20:08 +0200 Subject: [PATCH 32/34] fix(store): one process-wide section for a query connection's first access On an idle WAL database (no connection open, -shm reset) the first read of a new connection runs WAL recovery: walIndexRecover rebuilds the shared wal-index header under SQLite's write lock. A second connection of the same process opening at that moment reads that header lock-free at the start of walTryBeginRead, before it takes its read lock. Across processes SQLite's file locks order the two; between the daemon's request threads nothing did. TSan reported exactly that pair three times in CI (2026-10-06 on #2458's tsan-arm leg, 2026-10-07 twice on #2348's macOS leg), from concurrent index_repository requests reading the project record through cbm_store_open_path_query. cbm_store_open_path_query now opens and probes the database (the plain read-only open, its first read, and the immutable fallback) inside SQLITE_MUTEX_STATIC_APP1, the process-wide mutex SQLite reserves for the application. The probe is one sqlite_master read, so the section is short; the queries that follow run outside it, as before. Regression test store_query_first_access_is_one_process_section: a test seam holds one opener inside the section, and sqlite3_mutex_try on the section's mutex must then report SQLITE_BUSY. It fails without the lock (try returns SQLITE_OK) and passes with it. SQLite's Windows build reports BUSY from sqlite3_mutex_try unconditionally, so the failing case is shown on POSIX; the locking code is the same on every platform. Signed-off-by: Martin Vogel --- src/store/store.c | 99 ++++++++++++++++++++++++----------- src/store/store.h | 7 +++ tests/test_store_checkpoint.c | 65 +++++++++++++++++++++++ 3 files changed, 140 insertions(+), 31 deletions(-) diff --git a/src/store/store.c b/src/store/store.c index 375834a25..31737ea4f 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -990,33 +990,24 @@ static bool build_immutable_uri(const char *path, char *out, size_t out_sz) { return true; } -cbm_store_t *cbm_store_open_path_query(const char *db_path) { - if (!db_path) { - return NULL; - } - - cbm_store_t *s = calloc(CBM_ALLOC_ONE, sizeof(cbm_store_t)); - if (!s) { - return NULL; - } - - /* Query tools open the project DB READ-ONLY: a read query must never - * mutate the DB (the previous READWRITE open + WAL write-pragmas did), - * and must work on a read-only DB file / filesystem. - * - * Try a plain READONLY open first — on a normal writable filesystem this - * reads WAL frames correctly via the -shm wal-index. SQLite opens lazily, - * so a read-only-filesystem failure (cannot create -shm for a WAL-mode - * DB) surfaces on first access, not at open time; we probe with a trivial - * read to force it. If the probe fails, retry once with an immutable URI - * that bypasses WAL and reads the main DB file directly. - * - * No SQLITE_OPEN_CREATE on either path — a missing DB must return NULL - * (no ghost .db for unknown/unindexed projects). */ +/* Query tools open the project DB READ-ONLY: a read query must never + * mutate the DB (the previous READWRITE open + WAL write-pragmas did), + * and must work on a read-only DB file / filesystem. + * + * Try a plain READONLY open first — on a normal writable filesystem this + * reads WAL frames correctly via the -shm wal-index. SQLite opens lazily, + * so a read-only-filesystem failure (cannot create -shm for a WAL-mode + * DB) surfaces on first access, not at open time; we probe with a trivial + * read to force it. If the probe fails, retry once with an immutable URI + * that bypasses WAL and reads the main DB file directly. + * + * No SQLITE_OPEN_CREATE on either path — a missing DB must return NULL + * (no ghost .db for unknown/unindexed projects). Returns false with s->db + * closed when the DB cannot be opened. */ +static bool query_open_first_access(cbm_store_t *s, const char *db_path) { char open_path[4096]; if (!cbm_path_for_file_api(db_path, open_path, sizeof(open_path))) { - free(s); - return NULL; + return false; } int rc = sqlite3_open_v2(open_path, &s->db, SQLITE_OPEN_READONLY, NULL); if (rc == SQLITE_OK) { @@ -1036,22 +1027,68 @@ cbm_store_t *cbm_store_open_path_query(const char *db_path) { * be opened (the read-only-filesystem case). This also keeps the * common "project not found" path to a single open attempt. */ if (!cbm_file_exists(db_path)) { - free(s); - return NULL; + return false; } char uri[ST_QUERY_URI_MAX]; if (!build_immutable_uri(db_path, uri, sizeof(uri))) { - free(s); - return NULL; + return false; } rc = sqlite3_open_v2(uri, &s->db, SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, NULL); if (rc != SQLITE_OK) { /* sqlite3_open_v2 allocates a handle even on failure — must close it. */ sqlite3_close(s->db); - free(s); - return NULL; + s->db = NULL; + return false; } } + return true; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static void (*g_first_access_hook)(void *ctx) = NULL; +static void *g_first_access_hook_ctx = NULL; +void cbm_store_query_first_access_hook_for_testing(void (*hook)(void *ctx), void *ctx) { + g_first_access_hook = hook; + g_first_access_hook_ctx = ctx; +} +static void query_first_access_hook(void) { + if (g_first_access_hook) { + g_first_access_hook(g_first_access_hook_ctx); + } +} +#else +static void query_first_access_hook(void) {} +#endif + +cbm_store_t *cbm_store_open_path_query(const char *db_path) { + if (!db_path) { + return NULL; + } + + cbm_store_t *s = calloc(CBM_ALLOC_ONE, sizeof(cbm_store_t)); + if (!s) { + return NULL; + } + + /* One query connection at a time takes its first look at the WAL. On an + * idle DB (no connection open, -shm reset) that first read runs WAL + * recovery, rebuilding the shared wal-index header under SQLite's write + * lock, while another opener reads the same header lock-free before it + * takes its read lock (walTryBeginRead). Across processes SQLite's file + * locks order the two; between this process's request threads nothing + * did, and TSan reported walTryBeginRead against walIndexRecover from + * concurrent index_repository requests. Holding one process-wide mutex + * across open + first access puts a recovery before the next connection's + * first read. The probe is one sqlite_master read, so the hold is short. */ + sqlite3_mutex *first_access = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP1); + sqlite3_mutex_enter(first_access); + bool opened = query_open_first_access(s, db_path); + query_first_access_hook(); + sqlite3_mutex_leave(first_access); + if (!opened) { + free(s); + return NULL; + } s->db_path = heap_strdup(db_path); diff --git a/src/store/store.h b/src/store/store.h index fd690b5da..058e1a587 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -385,6 +385,13 @@ cbm_store_t *cbm_store_open_path_existing(const char *db_path); * exist — never creates a new .db file. */ cbm_store_t *cbm_store_open_path_query(const char *db_path); +#ifdef CBM_ENABLE_TEST_SEAMS +/* Every following query open calls `hook(ctx)` inside its first-access + * section (after the first read, the process-wide lock still held) until the + * hook is cleared with NULL. Test builds only. */ +void cbm_store_query_first_access_hook_for_testing(void (*hook)(void *ctx), void *ctx); +#endif + /* Validate and seal an existing DB for atomic replacement without creating or * migrating its schema. Returns OK when sealed, NOT_FOUND when the bytes are * definitely corrupt/incompatible and should be quarantined, or ERR when the diff --git a/tests/test_store_checkpoint.c b/tests/test_store_checkpoint.c index 8ab6d76ae..295ccd001 100644 --- a/tests/test_store_checkpoint.c +++ b/tests/test_store_checkpoint.c @@ -20,6 +20,8 @@ #include #include #include +#include "../src/foundation/compat_thread.h" +#include static void tsc_cleanup_db(const char *db_path) { char sidecar[512]; @@ -426,7 +428,70 @@ TEST(remove_db_sidecars_rejects_truncated_suffix_path) { PASS(); } +/* The first read of a query connection on an idle WAL DB runs WAL recovery, + * rebuilding the shared wal-index header; another connection of this process + * opening at the same moment read that header lock-free (TSan, three CI + * sightings: walTryBeginRead vs walIndexRecover from concurrent + * index_repository requests). A query open's first access is therefore one + * process-wide section, SQLite's SQLITE_MUTEX_STATIC_APP1. With opener A held + * inside it, the section must not be free. + * Windows: SQLite's sqlite3_mutex_try reports BUSY unconditionally without + * SQLITE_WIN32_MUTEX_TRYENTER, so there the assertion cannot go RED; the + * mutex is the same code on every platform and the RED proof is POSIX. */ +typedef struct { + const char *path; + atomic_int inside; + atomic_int release; +} tsc_first_access_t; + +static void tsc_first_access_hold(void *ctx) { + tsc_first_access_t *p = ctx; + atomic_store_explicit(&p->inside, 1, memory_order_release); + while (!atomic_load_explicit(&p->release, memory_order_acquire)) {} +} + +static void *tsc_first_access_open(void *arg) { + tsc_first_access_t *p = arg; + cbm_store_t *s = cbm_store_open_path_query(p->path); + if (s) { + cbm_store_close(s); + } + return NULL; +} + +TEST(store_query_first_access_is_one_process_section) { + char *dir = th_mktempdir("cbm_first_access"); + ASSERT_NOT_NULL(dir); + char db[512]; + snprintf(db, sizeof(db), "%s/g.db", dir); + cbm_store_t *w = cbm_store_open_path(db); + ASSERT_NOT_NULL(w); + cbm_store_close(w); + + tsc_first_access_t p = {.path = db}; + atomic_init(&p.inside, 0); + atomic_init(&p.release, 0); + cbm_store_query_first_access_hook_for_testing(tsc_first_access_hold, &p); + cbm_thread_t opener; + ASSERT_EQ(cbm_thread_create(&opener, 0, tsc_first_access_open, &p), 0); + while (!atomic_load_explicit(&p.inside, memory_order_acquire)) {} + sqlite3_mutex *section = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP1); + int try_rc = sqlite3_mutex_try(section); + if (try_rc == SQLITE_OK) { + sqlite3_mutex_leave(section); + } + atomic_store_explicit(&p.release, 1, memory_order_release); + (void)cbm_thread_join(&opener); + cbm_store_query_first_access_hook_for_testing(NULL, NULL); + tsc_cleanup_db(db); + th_rmtree(dir); + + ASSERT_EQ(try_rc, SQLITE_BUSY); + PASS(); +} + SUITE(store_checkpoint) { + RUN_TEST(store_query_first_access_is_one_process_section); RUN_TEST(checkpoint_does_not_truncate_wal); RUN_TEST(seal_for_atomic_publish_makes_main_file_self_contained); RUN_TEST(seal_for_atomic_publish_fails_closed_while_reader_pins_wal); From c828105d1eafe377e00a465953613eb0e32c0a31 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 22:16:36 +0200 Subject: [PATCH 33/34] fix(extract): write C variants in graph_buffer's schema Main's graph_buffer.c ("Definition variants") records a definition's variant spans as [{"file_path","start_line","end_line"}], and the test-impact engine reads that schema to find every TEST(...) branch of one test. This branch's extractor wrote [{"start","end"}]. On the merge with main, graph_buffer's same-QN merge found a non-empty variants list it could not read and dropped every span, the definition's own included. The engine then no longer found the second #if branch of TEST(alpha_uses_lib), so test_impact_engine_variant_tests_are_one_test reported suite alpha UNMAPPED, in CI on macOS and Windows and locally. The extractor now writes the shared schema, with the file path JSON-escaped. The C-preprocessor-only gate and the one-carrier rule are unchanged, and the variants assertions use the shared format. test_impact_engine, extraction, pipeline, graph_buffer and conditional_variants: 857 passed. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 25 +++++++++++++++++++++---- internal/cbm/cbm.h | 4 +++- tests/test_extraction.c | 9 ++++++--- tests/test_pipeline.c | 15 ++++++++++----- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index f4ce1354f..366a71408 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -26,12 +26,14 @@ #include "foundation/compat.h" #include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write #include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set +#include "foundation/str_util.h" // cbm_json_escape — variants file_path #include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState #include "foundation/constants.h" #include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free/mi_usable_size — bind 3rd-party allocators (#424) #if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR #include "sqlite3.h" // sqlite3_mem_methods, sqlite3_config, SQLITE_CONFIG_MALLOC — bind sqlite to mimalloc #endif +#include // INT_MAX #include // uint32_t, uint64_t, int64_t #include #include @@ -2165,7 +2167,9 @@ static uint64_t cbm_variant_key(const char *qn, const char *label) { /* refs[0..n) is one (QN, label) group in span order: give every member the * list of the group's distinct spans. */ static void cbm_variant_assign(CBMFileResult *result, const cbm_variant_ref_t *refs, int n) { - enum { VARIANT_ENTRY_MAX = 40 }; /* ,{"start":4294967295,"end":4294967295} */ + /* ,{"file_path":"","start_line":4294967295,"end_line":4294967295} + path; + * cbm_json_escape writes at most 6 bytes per source byte (\u00XX). */ + enum { VARIANT_ENTRY_FIXED = 64, JSON_ESCAPE_GROWTH = 6 }; int distinct = 0; for (int i = 0; i < n; i++) { if (i == 0 || refs[i].start != refs[i - 1].start || refs[i].end != refs[i - 1].end) { @@ -2175,7 +2179,17 @@ static void cbm_variant_assign(CBMFileResult *result, const cbm_variant_ref_t *r if (distinct < 2) { return; } - size_t cap = (size_t)distinct * VARIANT_ENTRY_MAX + 3; + const char *path = result->defs.items[refs[0].idx].file_path; + size_t path_cap = (path ? strlen(path) : 0) * JSON_ESCAPE_GROWTH + 1; + if (path_cap > INT_MAX) { + return; + } + char *esc = (char *)cbm_arena_alloc(&result->arena, path_cap); + if (!esc) { + return; + } + (void)cbm_json_escape(esc, (int)path_cap, path ? path : ""); + size_t cap = (size_t)distinct * (VARIANT_ENTRY_FIXED + strlen(esc)) + 3; char *json = (char *)cbm_arena_alloc(&result->arena, cap); if (!json) { return; @@ -2186,8 +2200,11 @@ static void cbm_variant_assign(CBMFileResult *result, const cbm_variant_ref_t *r if (i > 0 && refs[i].start == refs[i - 1].start && refs[i].end == refs[i - 1].end) { continue; } - int w = snprintf(json + pos, cap - pos, "%s{\"start\":%u,\"end\":%u}", pos > 1 ? "," : "", - refs[i].start, refs[i].end); + /* graph_buffer.c's variants schema, so its same-QN merge and every + * reader of the node (the test-impact engine) see one format. */ + int w = snprintf(json + pos, cap - pos, + "%s{\"file_path\":\"%s\",\"start_line\":%u,\"end_line\":%u}", + pos > 1 ? "," : "", esc, refs[i].start, refs[i].end); if (w <= 0 || (size_t)w >= cap - pos) { return; /* cannot happen: cap covers the longest entry */ } diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 598d10b56..1c3b0a64a 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -271,7 +271,9 @@ typedef struct { * (`#if`/`#else` twins, a macro redefined per platform, overloads): the * graph keeps one node per QN, and this lists every one of those * definitions' line spans, the surviving one included, as a JSON array - * sorted by start line: [{"start":10,"end":14},{"start":20,"end":26}]. + * sorted by start line, in graph_buffer.c's variants schema: + * [{"file_path":"a.c","start_line":10,"end_line":14}, + * {"file_path":"a.c","start_line":20,"end_line":26}]. * Carried by the definition the graph keeps for the file (the last by * start line), which makes it the node's `variants` property. NULL on * every other definition, and in every language outside diff --git a/tests/test_extraction.c b/tests/test_extraction.c index c5057e7f7..5c85441ea 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7743,15 +7743,18 @@ TEST(extract_c_variants_lists_same_file_duplicates_c1) { ASSERT_NOT_NULL(kept); ASSERT_NOT_NULL(other); ASSERT_NOT_NULL(kept->variants); - ASSERT_STR_EQ(kept->variants, "[{\"start\":2,\"end\":2},{\"start\":4,\"end\":6}]"); + ASSERT_STR_EQ(kept->variants, "[{\"file_path\":\"v.c\",\"start_line\":2,\"end_line\":2}," + "{\"file_path\":\"v.c\",\"start_line\":4,\"end_line\":6}]"); ASSERT_NULL(other->variants); /* one carrier per group: the node the graph keeps */ const CBMDefinition *lim_a = c1_def_at(r, "Macro", "LIMIT", 9); const CBMDefinition *lim_b = c1_def_at(r, "Macro", "LIMIT", 11); ASSERT_NOT_NULL(lim_a); ASSERT_NOT_NULL(lim_b); - char want[128]; - snprintf(want, sizeof(want), "[{\"start\":%u,\"end\":%u},{\"start\":%u,\"end\":%u}]", + char want[256]; + snprintf(want, sizeof(want), + "[{\"file_path\":\"v.c\",\"start_line\":%u,\"end_line\":%u}," + "{\"file_path\":\"v.c\",\"start_line\":%u,\"end_line\":%u}]", lim_a->start_line, lim_a->end_line, lim_b->start_line, lim_b->end_line); ASSERT_NOT_NULL(lim_b->variants); ASSERT_STR_EQ(lim_b->variants, want); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 135560799..5aeb49c1a 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10536,9 +10536,12 @@ static c1_variants_obs_t c1_observe_variants(const char *repo, const char *db_na return obs; } obs.opened = true; - obs.pick_lists_both = c1_props_contain( - s, proj, "v.pick", "\"variants\":[{\"start\":2,\"end\":2},{\"start\":4,\"end\":6}]"); - obs.limit_has_list = c1_props_contain(s, proj, "v.LIMIT#macro", "\"variants\":[{\"start\":9,"); + obs.pick_lists_both = + c1_props_contain(s, proj, "v.pick", + "\"variants\":[{\"file_path\":\"v.c\",\"start_line\":2,\"end_line\":2}," + "{\"file_path\":\"v.c\",\"start_line\":4,\"end_line\":6}]"); + obs.limit_has_list = c1_props_contain(s, proj, "v.LIMIT#macro", + "\"variants\":[{\"file_path\":\"v.c\",\"start_line\":9,"); obs.once_has_list = c1_props_contain(s, proj, "v.once", "\"variants\""); cbm_node_t *nodes = NULL; int count = 0; @@ -10680,8 +10683,10 @@ TEST(pipeline_cpp_overloads_are_listed_as_variants_c1) { int scale_nodes = n; int kept_start = n > 0 ? nodes[0].start_line : -1; cbm_store_free_nodes(nodes, n); - bool lists_both = c1_props_contain( - s, proj, "ov.scale", "\"variants\":[{\"start\":1,\"end\":1},{\"start\":2,\"end\":4}]"); + bool lists_both = + c1_props_contain(s, proj, "ov.scale", + "\"variants\":[{\"file_path\":\"ov.cpp\",\"start_line\":1,\"end_line\":1}," + "{\"file_path\":\"ov.cpp\",\"start_line\":2,\"end_line\":4}]"); bool once_has_list = c1_props_contain(s, proj, "ov.once", "\"variants\""); cbm_store_close(s); From 8a97baabcb5d4a52f2c2439ee58535aaa548db4a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 8 Oct 2026 01:37:03 +0200 Subject: [PATCH 34/34] fix(extract): parse the first-branch projection under the terminated-last-line rule Merging main brought #2340's virtual_newline field in CBMStringInput. cbm_rescue_defs_from_projection still built its input with the two-field initializer, a missing-field-initializers error under -Werror. The projection keeps the raw source's length and line structure, so it now goes through cbm_parse_source like the raw parse and the C++ branch views: the same virtual newline, and the same edit that removes it again. extraction pipeline graph_buffer store_search mcp registry edge_types_probe parse_coverage c_lsp test_impact_engine conditional_variants on the merge result: 2277 passed, 4 skipped. Signed-off-by: Martin Vogel --- internal/cbm/cbm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 3dab239d9..e0d5cdd61 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -2571,10 +2571,10 @@ static void cbm_rescue_defs_from_projection(CBMExtractCtx *raw_ctx, const TSLang TSTree *tree = NULL; if (parser) { ts_parser_reset(parser); - CBMStringInput input = {projected, (uint32_t)raw_ctx->source_len}; - TSInput ts_input = {&input, cbm_string_read, TSInputEncodingUTF8, NULL}; TSParseOptions opts = {0}; - tree = ts_parser_parse_with_options(parser, NULL, ts_input, opts); + /* The projection keeps the raw source's length and line structure, + * so it is parsed under the same terminated-last-line rule (#2078). */ + tree = cbm_parse_source(parser, projected, (uint32_t)raw_ctx->source_len, opts); } CBMHashTable *held = tree ? cbm_ht_create(CBM_SZ_256) : NULL; if (held) {