nprasad2077/codebase-memory-mcp
1
0 Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-04 10:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2013 from shafty023/fix/1997-extract-scratch-arena

Browse Source 
fix(extract): AST traversal stacks no longer outlive the file they walk
This commit is contained in:
Martin Vogel authored and GitHub committed 2026-09-03 08:14:16 +02:00
2 parents 0e9ed75991 40f2722d61
commit db2f104a1f
12 files changed
+359 -186
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/cbm/cbm.c
+47 -5
View File
@@ -1,3 +1,7 @@
/* Full declaration set for the same CBMArena, and it must precede cbm.h:
* internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H
* guard, so whichever is included first is the one this file sees. */
#include "foundation/arena.h" // cbm_arena_init_sized
#include "cbm.h"
#include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy
#include "helpers.h"
@@ -1186,11 +1190,27 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
return r;
}
CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path,
int64_t timeout_micros, const char **extra_defines,
const char **include_paths, const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table) {
/* Initial block for the per-file traversal scratch arena, chosen by measuring
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
* both this size and at 1 MB, and on most files at 256 KB, where the two
* channel walks alone are exactly 262144 bytes. 512 KB therefore buys the same
* growth behaviour as 1 MB for half the resident block per worker. It is also
* exactly MI_LARGE_MAX_OBJ_SIZE in the vendored mimalloc
* (vendored/mimalloc/include/mimalloc/types.h:426, MI_LARGE_PAGE_SIZE/8 with
* MI_ENABLE_LARGE_PAGES defaulting to 1 at :115 and not overridden here), so
* the block is still bin-allocated from a large page. Growth is not free at
* this size for the same reason: arena_grow doubles to 1 MiB, which is above
* that bound and so a singleton OS allocation. One file in twelve thousand
* pays it, which is why the cost is accepted. */
enum { CBM_EXTRACT_SCRATCH_BLOCK = CBM_SZ_512 * CBM_SZ_1K };
static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path,
int64_t timeout_micros, const char **extra_defines,
const char **include_paths,
const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table,
CBMArena *scratch) {
// Allocate result on heap (arena inside for all string data)
enum { SINGLE = 1 };
CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
@@ -1301,6 +1321,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
// Build extraction context
CBMExtractCtx ctx = {
.arena = a,
.scratch = scratch,
.result = result,
.source = source,
.source_len = source_len,
@@ -1429,6 +1450,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
// Build context for expanded source — extract only calls via unified extractor
CBMExtractCtx pp_ctx = {
.arena = a,
.scratch = scratch,
.result = result,
.source = expanded,
.source_len = expanded_len,
@@ -1665,6 +1687,26 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
return result;
}
/* Public entry. Owns the traversal scratch arena for the whole of one file's
* extraction: created here, handed to the body as ctx->scratch, destroyed on
* the way out. The body has seven early returns, so bracketing it in a wrapper
* is what keeps that to one create and one destroy. If the arena cannot be
* created, the body is handed NULL and the traversal stacks fall back to the
* result arena, which is what shipped before #1997. */
CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path,
int64_t timeout_micros, const char **extra_defines,
const char **include_paths, const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table) {
CBMArena scratch;
cbm_arena_init_sized(&scratch, CBM_EXTRACT_SCRATCH_BLOCK);
CBMFileResult *result = extract_file_ex_body(
source, source_len, language, project, rel_path, timeout_micros, extra_defines,
include_paths, macro_table, return_type_table, scratch.nblocks > 0 ? &scratch : NULL);
cbm_arena_destroy(&scratch);
return result;
}
void cbm_free_result(CBMFileResult *result) {
if (!result) {
return;
internal/cbm/cbm.h
+7
View File
@@ -589,6 +589,13 @@ typedef struct {
typedef struct {
CBMArena *arena;
/* Scratch for AST traversal, owned by the cbm_extract_file_ex call that
* built this context and destroyed when it returns. Nothing a
* CBMFileResult points at may be allocated here: `arena` is the result's
* own, and it outlives extraction by the whole pipeline (#1997). NULL in a
* context built without one, in which case the stacks fall back to
* `arena`. */
CBMArena *scratch;
CBMFileResult *result;
const char *source;
int source_len;
internal/cbm/extract_channels.c
+30 -30
View File
@@ -101,8 +101,8 @@ static const char *literal_from_first_child(CBMExtractCtx *ctx, TSNode node) {
* const table per file is sufficient for the common Socket.IO pattern). */
static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) {
TSNode node = ts_nstack_pop(&stack);
@@ -128,15 +128,15 @@ static void scan_string_consts_js(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
}
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
/* Python constant resolution: NAME = "value" (assignment node). */
static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tbl) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0 && tbl->count < CHAN_CONST_CAP) {
TSNode node = ts_nstack_pop(&stack);
@@ -163,7 +163,7 @@ static void scan_string_consts_python(CBMExtractCtx *ctx, chan_const_table_t *tb
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -373,15 +373,15 @@ static void extract_channels_js(CBMExtractCtx *ctx) {
/* Second pass: walk the tree looking for call_expression nodes. */
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "call_expression") == 0) {
js_process_call(ctx, node, &consts);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -549,8 +549,8 @@ static void extract_channels_python(CBMExtractCtx *ctx) {
scan_string_consts_python(ctx, &consts);
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -562,7 +562,7 @@ static void extract_channels_python(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -627,8 +627,8 @@ static void go_process_call(CBMExtractCtx *ctx, TSNode call) {
static void extract_channels_go(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -637,7 +637,7 @@ static void extract_channels_go(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -724,8 +724,8 @@ static void java_process_annotation(CBMExtractCtx *ctx, TSNode annotation) {
static void extract_channels_java(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -737,7 +737,7 @@ static void extract_channels_java(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -793,8 +793,8 @@ static void csharp_process_call(CBMExtractCtx *ctx, TSNode call) {
static void extract_channels_csharp(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -803,7 +803,7 @@ static void extract_channels_csharp(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -856,8 +856,8 @@ static void ruby_process_call(CBMExtractCtx *ctx, TSNode call) {
static void extract_channels_ruby(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -866,7 +866,7 @@ static void extract_channels_ruby(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -958,8 +958,8 @@ static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) {
static void extract_channels_elixir(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -971,7 +971,7 @@ static void extract_channels_elixir(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -1034,8 +1034,8 @@ static void rust_process_call(CBMExtractCtx *ctx, TSNode call) {
static void extract_channels_rust(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CHAN_STACK_CAP);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -1044,7 +1044,7 @@ static void extract_channels_rust(CBMExtractCtx *ctx) {
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
internal/cbm/extract_defs.c
+13 -13
View File
@@ -5316,8 +5316,8 @@ static TSNode emit_elixir_module_class(CBMExtractCtx *ctx, TSNode cur) {
static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) {
(void)spec;
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_64);
ts_nstack_push(&stack, ctx->arena, node);
ts_nstack_init(&stack, ctx, CBM_SZ_64);
ts_nstack_push(&stack, node);
while (stack.count > 0) {
TSNode cur = ts_nstack_pop(&stack);
@@ -5345,7 +5345,7 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp
for (int di = (int)dbc - SKIP_CHAR; di >= 0; di--) {
TSNode dchild = ts_node_child(do_block, (uint32_t)di);
if (!ts_node_is_null(dchild) && strcmp(ts_node_type(dchild), "call") == 0) {
ts_nstack_push(&stack, ctx->arena, dchild);
ts_nstack_push(&stack, dchild);
}
}
}
@@ -6285,8 +6285,8 @@ static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
// Used by YAML, TOML, INI, JSON.
static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_256);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, CBM_SZ_256);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -6310,7 +6310,7 @@ static void walk_variables_iter(CBMExtractCtx *ctx, TSNode root, const CBMLangSp
strcmp(ck, "section") == 0 || strcmp(ck, "object") == 0 ||
strcmp(ck, "array") == 0 || strcmp(ck, "pair") == 0 || strcmp(ck, "element") == 0 ||
strcmp(ck, "content") == 0) {
ts_nstack_push(&stack, ctx->arena, child);
ts_nstack_push(&stack, child);
}
}
}
@@ -7144,10 +7144,10 @@ static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enc
// Push nested class nodes from a class body container onto the defs stack.
// Iteratively walks into wrapper nodes (field_declaration, template_declaration).
static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s,
const char *enclosing_qn, CBMArena *arena) {
const char *enclosing_qn, const CBMExtractCtx *ctx) {
TSNodeStack nc_stack;
ts_nstack_init(&nc_stack, arena, NESTED_CLASS_STACK_CAP);
ts_nstack_push(&nc_stack, arena, body);
ts_nstack_init(&nc_stack, ctx, NESTED_CLASS_STACK_CAP);
ts_nstack_push(&nc_stack, body);
while (nc_stack.count > 0) {
TSNode cur = ts_nstack_pop(&nc_stack);
@@ -7162,7 +7162,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta
const char *ck = ts_node_type(child);
if (strcmp(ck, "field_declaration") == 0 ||
strcmp(ck, "template_declaration") == 0 || strcmp(ck, "declaration") == 0) {
ts_nstack_push(&nc_stack, arena, child);
ts_nstack_push(&nc_stack, child);
}
}
}
@@ -7278,7 +7278,7 @@ static void extract_typescript_namespace_def(CBMExtractCtx *ctx, TSNode node,
// Push nested class children from a class body container onto the walk stack.
static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_stack_t *s,
const char *new_enclosing, CBMArena *arena) {
const char *new_enclosing, const CBMExtractCtx *ctx) {
/* Use the same language-aware body selection as method extraction. The old
* independent spelling list omitted valid containers such as Scala's
* `template_body` and Solidity's contract body. Methods were extracted
@@ -7294,7 +7294,7 @@ static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_st
body = find_class_member_body(node, spec->language);
}
if (!ts_node_is_null(body)) {
push_nested_class_nodes(body, spec, s, new_enclosing, arena);
push_nested_class_nodes(body, spec, s, new_enclosing, ctx);
return;
}
@@ -7880,7 +7880,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
if (cbm_kind_in_set(node, spec->class_node_types)) {
extract_class_def(ctx, node, spec);
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
push_class_body_children(node, spec, &s, new_enclosing, ctx->arena);
push_class_body_children(node, spec, &s, new_enclosing, ctx);
continue;
}
internal/cbm/extract_env_accesses.c
+3 -3
View File
@@ -140,8 +140,8 @@ static bool is_env_var_name(const char *s) {
// Iterative env access walker — explicit stack
static void walk_env_accesses(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, 4096);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, 4096);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -163,7 +163,7 @@ static void walk_env_accesses(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec
continue; // don't push children (avoid double-counting)
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
internal/cbm/extract_imports.c
+108 -109
View File
@@ -506,8 +506,8 @@ static bool process_commonjs_require(CBMExtractCtx *ctx, TSNode call) {
static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -540,7 +540,7 @@ static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) {
}
if (push_children) {
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
}
@@ -1093,8 +1093,8 @@ static void parse_haskell_imports(CBMExtractCtx *ctx) {
static void parse_zig_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "builtin_function") == 0) {
@@ -1111,7 +1111,7 @@ static void parse_zig_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -1163,8 +1163,8 @@ static void process_wolfram_needs(CBMExtractCtx *ctx, TSNode node) {
static void walk_wolfram_imports(CBMExtractCtx *ctx, TSNode root) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -1176,7 +1176,7 @@ static void walk_wolfram_imports(CBMExtractCtx *ctx, TSNode root) {
process_wolfram_needs(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -1666,8 +1666,8 @@ static void capture_namespace_decl(CBMExtractCtx *ctx) {
static void parse_hare_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "use_statement") == 0) {
@@ -1686,7 +1686,7 @@ static void parse_hare_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -1698,8 +1698,8 @@ static void parse_hare_imports(CBMExtractCtx *ctx) {
static void parse_pascal_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "declUses") == 0) {
@@ -1717,7 +1717,7 @@ static void parse_pascal_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -1730,8 +1730,8 @@ static void parse_pascal_imports(CBMExtractCtx *ctx) {
static void parse_powershell_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "command") == 0) {
@@ -1741,8 +1741,8 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) {
/* Find the last generic_token anywhere under the command — that
* is the module path / namespace / assembly being imported. */
TSNodeStack inner;
ts_nstack_init(&inner, a, CBM_SZ_512);
ts_nstack_push(&inner, a, node);
ts_nstack_init(&inner, ctx, CBM_SZ_512);
ts_nstack_push(&inner, node);
const char *last_tok = NULL;
uint32_t last_start = 0;
while (inner.count > 0) {
@@ -1756,7 +1756,7 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) {
last_start = sb;
}
}
ts_nstack_push_children(&inner, a, c);
ts_nstack_push_children(&inner, c);
}
if (last_tok && last_tok[0]) {
CBMImport imp = {.local_name = path_last(a, last_tok),
@@ -1766,7 +1766,7 @@ static void parse_powershell_imports(CBMExtractCtx *ctx) {
}
continue; /* commands don't nest imports further */
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -1993,17 +1993,16 @@ static void lisp_process_list(CBMExtractCtx *ctx, TSNode node) {
* just root children. Stack-based (not recursive) to avoid deep-nesting stack
* overflow, matching the other walkers in this file. */
static void parse_lisp_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *nt = ts_node_type(node);
if (strcmp(nt, "list") == 0 || strcmp(nt, "list_lit") == 0) {
lisp_process_list(ctx, node);
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2013,8 +2012,8 @@ static void parse_lisp_imports(CBMExtractCtx *ctx) {
static void parse_starlark_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "call") == 0) {
@@ -2042,7 +2041,7 @@ static void parse_starlark_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2053,8 +2052,8 @@ static void parse_starlark_imports(CBMExtractCtx *ctx) {
static void parse_tcl_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "command") == 0) {
@@ -2076,7 +2075,7 @@ static void parse_tcl_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2087,8 +2086,8 @@ static void parse_tcl_imports(CBMExtractCtx *ctx) {
static void parse_teal_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "function_call") == 0) {
@@ -2112,7 +2111,7 @@ static void parse_teal_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2122,8 +2121,8 @@ static void parse_teal_imports(CBMExtractCtx *ctx) {
static void parse_zsh_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "command") == 0) {
@@ -2143,7 +2142,7 @@ static void parse_zsh_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2230,15 +2229,15 @@ static void parse_html_imports(CBMExtractCtx *ctx) {
parse_embedded_imports(ctx);
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "start_tag") == 0 ||
strcmp(ts_node_type(node), "self_closing_tag") == 0) {
html_extract_tag_src(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2297,8 +2296,8 @@ static void push_string_descendant_import(CBMExtractCtx *ctx, TSNode node) {
static void parse_cmake_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "normal_command") == 0) {
@@ -2316,15 +2315,15 @@ static void parse_cmake_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
// --- BitBake imports: require/include path ---
static void parse_bitbake_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2332,7 +2331,7 @@ static void parse_bitbake_imports(CBMExtractCtx *ctx) {
strcmp(k, "inherit_directive") == 0) {
push_string_descendant_import(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2344,8 +2343,8 @@ static void parse_bitbake_imports(CBMExtractCtx *ctx) {
static void parse_meson_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2368,35 +2367,35 @@ static void parse_meson_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
// --- Kconfig imports: source "path" ---
static void parse_kconfig_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (ts_node_is_named(node) && strcmp(ts_node_type(node), "source") == 0) {
push_string_descendant_import(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
// --- GN imports: import("//path") ---
static void parse_gn_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "import_statement") == 0) {
push_string_descendant_import(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2420,8 +2419,8 @@ static void parse_just_imports(CBMExtractCtx *ctx) {
static void parse_nix_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "apply_expression") == 0) {
@@ -2440,15 +2439,15 @@ static void parse_nix_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
// --- Jsonnet imports: import 'path' / importstr 'path' ---
static void parse_jsonnet_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2456,15 +2455,15 @@ static void parse_jsonnet_imports(CBMExtractCtx *ctx) {
strcmp(k, "importbin") == 0)) {
push_string_descendant_import(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
// --- Pkl imports: amends/extends/import "path" ---
static void parse_pkl_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2472,7 +2471,7 @@ static void parse_pkl_imports(CBMExtractCtx *ctx) {
strcmp(k, "importExpr") == 0) {
push_string_descendant_import(ctx, node);
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2480,8 +2479,8 @@ static void parse_pkl_imports(CBMExtractCtx *ctx) {
static void parse_nickel_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
/* The grammar emits an anonymous `import` token followed by a
@@ -2502,7 +2501,7 @@ static void parse_nickel_imports(CBMExtractCtx *ctx) {
}
}
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2526,8 +2525,8 @@ static void parse_thrift_imports(CBMExtractCtx *ctx) {
// --- Cap'n Proto imports: using X = import "path" ---
static void parse_capnp_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2536,7 +2535,7 @@ static void parse_capnp_imports(CBMExtractCtx *ctx) {
push_string_descendant_import(ctx, node);
continue; /* don't double-emit from nested import_path */
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2544,8 +2543,8 @@ static void parse_capnp_imports(CBMExtractCtx *ctx) {
static void parse_dlang_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "import_declaration") == 0) {
@@ -2559,15 +2558,15 @@ static void parse_dlang_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
// --- TableGen imports: include "path.td" ---
static void parse_tablegen_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2576,22 +2575,22 @@ static void parse_tablegen_imports(CBMExtractCtx *ctx) {
push_string_descendant_import(ctx, node);
continue;
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
// --- Crystal imports: require "./path" ---
static void parse_crystal_imports(CBMExtractCtx *ctx) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (ts_node_is_named(node) && strcmp(ts_node_type(node), "require") == 0) {
push_string_descendant_import(ctx, node);
continue;
}
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2599,8 +2598,8 @@ static void parse_crystal_imports(CBMExtractCtx *ctx) {
static void parse_fsharp_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2616,7 +2615,7 @@ static void parse_fsharp_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2624,8 +2623,8 @@ static void parse_fsharp_imports(CBMExtractCtx *ctx) {
static void parse_ada_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "with_clause") == 0) {
@@ -2644,7 +2643,7 @@ static void parse_ada_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2655,8 +2654,8 @@ static void parse_ada_imports(CBMExtractCtx *ctx) {
static void parse_elm_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "import_clause") == 0) {
@@ -2678,7 +2677,7 @@ static void parse_elm_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2689,8 +2688,8 @@ static void parse_elm_imports(CBMExtractCtx *ctx) {
static void parse_move_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "use_declaration") == 0) {
@@ -2707,7 +2706,7 @@ static void parse_move_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2741,8 +2740,8 @@ static char *smali_demangle_descriptor(CBMArena *a, const char *desc) {
static void parse_smali_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2757,7 +2756,7 @@ static void parse_smali_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2767,8 +2766,8 @@ static void parse_smali_imports(CBMExtractCtx *ctx) {
static void parse_tlaplus_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
const char *k = ts_node_type(node);
@@ -2787,7 +2786,7 @@ static void parse_tlaplus_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2799,8 +2798,8 @@ static void parse_tlaplus_imports(CBMExtractCtx *ctx) {
static void parse_vhdl_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "use_clause") == 0) {
@@ -2818,7 +2817,7 @@ static void parse_vhdl_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2828,8 +2827,8 @@ static void parse_vhdl_imports(CBMExtractCtx *ctx) {
static void parse_wit_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "use_item") == 0) {
@@ -2847,7 +2846,7 @@ static void parse_wit_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2858,8 +2857,8 @@ static void parse_wit_imports(CBMExtractCtx *ctx) {
static void parse_smithy_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "use_statement") == 0) {
@@ -2881,7 +2880,7 @@ static void parse_smithy_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
@@ -2891,8 +2890,8 @@ static void parse_smithy_imports(CBMExtractCtx *ctx) {
static void parse_hyprlang_imports(CBMExtractCtx *ctx) {
CBMArena *a = ctx->arena;
TSNodeStack stack;
ts_nstack_init(&stack, a, CBM_SZ_512);
ts_nstack_push(&stack, a, ctx->root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, ctx->root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (strcmp(ts_node_type(node), "source") == 0) {
@@ -2909,7 +2908,7 @@ static void parse_hyprlang_imports(CBMExtractCtx *ctx) {
}
continue;
}
ts_nstack_push_children(&stack, a, node);
ts_nstack_push_children(&stack, node);
}
}
internal/cbm/extract_node_stack.h
+30 -11
View File
@@ -4,14 +4,22 @@
* Replaces fixed-size TSNode stack[] arrays that silently drop AST subtrees
* when the stack overflows (GitHub issue #199).
*
* Uses the arena allocator for zero-fragmentation growth: old blocks are
* abandoned (freed when the arena is destroyed at end of file extraction).
* Initial capacity matches the previous fixed caps so small files allocate
* no extra memory.
* Traversal stacks are scratch: nothing in a CBMFileResult ever points into
* one. They are therefore cut from ctx->scratch, which the enclosing
* cbm_extract_file_ex call owns and destroys on the way out, and never from
* ctx->arena, which the pipeline holds for every file until the whole result
* cache is freed (#1997).
*
* Growth abandons the old buffer in that scratch arena, which is free because
* the arena dies with the file. The initial capacities below are unchanged, but
* they are no longer what a small file costs: cbm_extract_file_ex creates the
* scratch arena's first block for every call, including calls that build no
* stack at all, and reclaims it on the way out.
*/
#ifndef CBM_EXTRACT_NODE_STACK_H
#define CBM_EXTRACT_NODE_STACK_H
#include "cbm.h" /* CBMExtractCtx: a stack draws from ctx->scratch */
#include "arena.h"
#include "tree_sitter/api.h"
#include <string.h> /* memcpy */
@@ -20,20 +28,31 @@ typedef struct {
TSNode *items;
int count;
int cap;
/* The arena every allocation for this stack comes from, recorded once by
* ts_nstack_init so push() cannot be handed a different one. It is
* ctx->scratch, or ctx->arena as the fallback when the context has none. */
CBMArena *scratch;
} TSNodeStack;
/* Initialize a stack with the given initial capacity, arena-allocated. */
static inline void ts_nstack_init(TSNodeStack *s, CBMArena *arena, int initial_cap) {
/* Initialize a stack with the given initial capacity, allocated from the
* context's traversal scratch. Taking the context rather than an arena is
* deliberate: it makes handing over ctx->arena, or a local alias of it, a type
* error rather than a retention bug nobody notices. A context built without a
* scratch falls back to ctx->arena, which is the behaviour that shipped before
* #1997, so no caller ever gets a NULL arena and silently loses nodes. */
static inline void ts_nstack_init(TSNodeStack *s, const CBMExtractCtx *ctx, int initial_cap) {
CBMArena *arena = ctx->scratch ? ctx->scratch : ctx->arena;
s->scratch = arena;
s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode));
s->count = 0;
s->cap = s->items ? initial_cap : 0;
}
/* Push a node onto the stack, growing 2x if needed. */
static inline void ts_nstack_push(TSNodeStack *s, CBMArena *arena, TSNode node) {
static inline void ts_nstack_push(TSNodeStack *s, TSNode node) {
if (s->count >= s->cap) {
int new_cap = s->cap ? s->cap * 2 : 512;
TSNode *new_items = (TSNode *)cbm_arena_alloc(arena, (size_t)new_cap * sizeof(TSNode));
TSNode *new_items = (TSNode *)cbm_arena_alloc(s->scratch, (size_t)new_cap * sizeof(TSNode));
if (!new_items)
return; /* OOM: best-effort, stop growing */
if (s->items && s->count > 0) {
@@ -54,7 +73,7 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) {
/*
* Push all children of `node` so they POP in forward (source) order — a drop-in
* replacement for the common idiom:
* for (int i = (int)count - 1; i >= 0; i--) ts_nstack_push(s, a, ts_node_child(node, i));
* for (int i = (int)count - 1; i >= 0; i--) ts_nstack_push(s, ts_node_child(node, i));
*
* That idiom calls ts_node_child(node, i) once per index, and ts_node_child is
* O(i) in tree-sitter (it walks the child iterator from the first child each
@@ -63,12 +82,12 @@ static inline TSNode ts_nstack_pop(TSNodeStack *s) {
* files). This helper enumerates children in a single O(N) cursor pass, then
* reverses the just-pushed segment so pop order is identical to the old idiom.
*/
static inline void ts_nstack_push_children(TSNodeStack *s, CBMArena *arena, TSNode node) {
static inline void ts_nstack_push_children(TSNodeStack *s, TSNode node) {
int base = s->count;
TSTreeCursor cursor = ts_tree_cursor_new(node);
if (ts_tree_cursor_goto_first_child(&cursor)) {
do {
ts_nstack_push(s, arena, ts_tree_cursor_current_node(&cursor));
ts_nstack_push(s, ts_tree_cursor_current_node(&cursor));
} while (ts_tree_cursor_goto_next_sibling(&cursor));
}
ts_tree_cursor_delete(&cursor);
internal/cbm/extract_semantic.c
+6 -6
View File
@@ -125,14 +125,14 @@ static void process_throw_node(CBMExtractCtx *ctx, TSNode node, const CBMLangSpe
// Iterative throw walker
static void walk_throws(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
process_throw_node(ctx, node, spec);
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - LAST_IDX; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -271,8 +271,8 @@ static void try_emit_assignment_write(CBMExtractCtx *ctx, TSNode node, const cha
static void walk_readwrites(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, CBM_SZ_512);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, CBM_SZ_512);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
if (cbm_kind_in_set(node, spec->assignment_node_types)) {
@@ -280,7 +280,7 @@ static void walk_readwrites(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - LAST_IDX; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
internal/cbm/extract_type_assigns.c
+3 -3
View File
@@ -186,12 +186,12 @@ static void process_type_assign_node(CBMExtractCtx *ctx, TSNode node, const CBML
// Walk AST for assignment patterns where RHS is a constructor call.
static void walk_type_assigns(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, 4096);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, 4096);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
process_type_assign_node(ctx, node, spec, cbm_enclosing_func_qn_cached(ctx, node));
ts_nstack_push_children(&stack, ctx->arena, node);
ts_nstack_push_children(&stack, node);
}
}
internal/cbm/extract_type_refs.c
+6 -6
View File
@@ -246,14 +246,14 @@ static void process_body_type_ref(CBMExtractCtx *ctx, TSNode node, const char *f
// Walk function body for type references (casts, type assertions, local var types, generics).
static void walk_body_type_refs(CBMExtractCtx *ctx, TSNode root, const char *func_qn) {
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, 4096);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, 4096);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
process_body_type_ref(ctx, node, func_qn);
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
@@ -290,8 +290,8 @@ static void walk_type_refs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *s
}
TSNodeStack stack;
ts_nstack_init(&stack, ctx->arena, 4096);
ts_nstack_push(&stack, ctx->arena, root);
ts_nstack_init(&stack, ctx, 4096);
ts_nstack_push(&stack, root);
while (stack.count > 0) {
TSNode node = ts_nstack_pop(&stack);
@@ -302,7 +302,7 @@ static void walk_type_refs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *s
}
uint32_t count = ts_node_child_count(node);
for (int i = (int)count - SKIP_ONE; i >= 0; i--) {
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
ts_nstack_push(&stack, ts_node_child(node, (uint32_t)i));
}
}
}
tests/test_extraction.c
+54
View File
@@ -7,6 +7,7 @@
*/
#include "test_framework.h"
#include "cbm.h"
#include "foundation/constants.h" /* CBM_SZ_* */
#include "../src/foundation/compat.h" /* cbm_clock_gettime (wide-flat scaling guard) */
#include "../src/foundation/compat_fs.h"
#include <time.h>
@@ -166,6 +167,57 @@ TEST(extract_ts_factory_object_methods_issue341) {
PASS();
}
/* #2010, split out of #1997: AST traversal stacks were allocated from
* result->arena, which the parallel pass holds for every file until the whole
* result cache is freed, so a one-file scratch structure was retained for the
* length of the index. cbm_extract_channels runs for every file and dispatches
* TypeScript to extract_channels_js, whose two walks take a 4096-entry TSNode
* stack each, and the ES import walk takes a 512-entry one:
* 2 * 4096 * 32 + 512 * 32 = 278528 bytes charged to the arena of a one-line
* file.
*
* The bound is derived, not tuned. Measured on this source, total_alloc was
* 365984 before the scratch arena and is 87456 after, exactly that difference.
* Of the 87456 that remain, 7680 is the defs item array at GROW_ARRAY's
* starting capacity of 32 times sizeof(CBMDefinition) 240, and the other 79776
* is everything else this file's extraction interns; none of it is traversal
* scratch. So the bound sits above 87456 with room and a factor of four below
* 365984.
*
* It is a byte budget, not a proof of lifetime; that is
* extract_traversal_stacks_come_from_ctx_scratch_issue2010 in test_mem.c. */
TEST(traversal_stack_not_in_result_arena_issue2010) {
CBMFileResult *r = extract("export const x = 1;\n", CBM_LANG_TYPESCRIPT, "t", "a.ts");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_TRUE(has_def_any(r, "x"));
/* Read the field rather than cbm_arena_total(): this file sees
* internal/cbm/arena.h, which declares a subset of the API. test_mem.c
* includes foundation/arena.h ahead of cbm.h and can call the accessor. */
ASSERT_LT(r->arena.total_alloc, (size_t)CBM_SZ_128 * CBM_SZ_1K);
cbm_free_result(r);
PASS();
}
/* Not a scratch test. The C and C++ preprocessed second pass builds its own
* extraction context (pp_ctx in cbm_extract_file_ex), and that context carries
* ctx->scratch so every context in the file is uniform, but nothing reads it
* there: pp_ctx reaches only cbm_extract_unified and cbm_run_c_lsp, and neither
* extract_unified.c nor anything under internal/cbm/lsp/ includes
* extract_node_stack.h, so no traversal stack is built on that path today.
* This guards the macro-expansion path itself, which had no assertion on a call
* that exists only after expansion. */
TEST(extract_c_macro_hidden_call_survives_preprocessed_pass_issue2010) {
CBMFileResult *r = extract("void target(void) {}\n"
"#define INVOKE() target()\n"
"void caller(void) { INVOKE(); }\n",
CBM_LANG_C, "t", "macro_call.c");
ASSERT_NOT_NULL(r);
ASSERT_TRUE(has_call(r, "target"));
cbm_free_result(r);
PASS();
}
/* --- C/C++ preprocessor macros become Macro nodes (#375) --- */
TEST(extract_c_macros_issue375) {
CBMFileResult *r = extract("#define SIMPLE_MACRO 1\n"
@@ -6665,6 +6717,8 @@ SUITE(extraction) {
RUN_TEST(extract_r_box_use_imports_issue218);
RUN_TEST(extract_r_dollar_call_issue219);
RUN_TEST(extract_ts_factory_object_methods_issue341);
RUN_TEST(traversal_stack_not_in_result_arena_issue2010);
RUN_TEST(extract_c_macro_hidden_call_survives_preprocessed_pass_issue2010);
RUN_TEST(extract_c_macros_issue375);
RUN_TEST(extract_cpp_macros_issue375);
RUN_TEST(extract_cpp_functionlike_macro_type_arg_no_false_parse_partial_issue1071);
tests/test_mem.c
+52
View File
@@ -14,6 +14,8 @@
#include "graph_buffer/graph_buffer.h"
#include "discover/discover.h"
#include "cbm.h"
#include "lang_specs.h" /* cbm_ts_language */
#include "foundation/constants.h" /* CBM_SZ_* */
#include <stdatomic.h>
#include <stdint.h>
@@ -1287,6 +1289,53 @@ TEST(mem_map_attributes_a_known_allocation) {
PASS();
}
/* #2010, the lifetime half. traversal_stack_not_in_result_arena in
* test_extraction.c pins the byte budget of the result arena, but a smaller
* CHAN_STACK_CAP would satisfy that too. This pins where the bytes actually
* went: the scratch arena takes the two 128 KB channel walks and the result
* arena does not. Builds the extraction context directly, since a completed
* cbm_extract_file_ex has already destroyed its scratch. */
TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010) {
enum { CHANNEL_WALK_BYTES = 2 * 4096 * (int)sizeof(TSNode) };
const char *src = "export const x = 1;\n";
const TSLanguage *ts_lang = cbm_ts_language(CBM_LANG_TYPESCRIPT);
ASSERT_NOT_NULL((void *)ts_lang);
TSParser *parser = ts_parser_new();
ASSERT_NOT_NULL(parser);
ts_parser_set_language(parser, ts_lang);
TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src));
ASSERT_NOT_NULL(tree);
CBMFileResult result;
memset(&result, 0, sizeof(result));
cbm_arena_init(&result.arena);
CBMArena scratch;
cbm_arena_init_sized(&scratch, (size_t)CBM_SZ_512 * CBM_SZ_1K);
CBMExtractCtx ctx = {
.arena = &result.arena,
.scratch = &scratch,
.result = &result,
.source = src,
.source_len = (int)strlen(src),
.language = CBM_LANG_TYPESCRIPT,
.project = "t",
.rel_path = "a.ts",
.root = ts_tree_root_node(tree),
};
cbm_extract_channels(&ctx);
ASSERT_GTE(cbm_arena_total(&scratch), (size_t)CHANNEL_WALK_BYTES);
ASSERT_LT(cbm_arena_total(&result.arena), (size_t)CBM_SZ_64 * CBM_SZ_1K);
cbm_arena_destroy(&scratch);
cbm_arena_destroy(&result.arena);
ts_tree_delete(tree);
ts_parser_delete(parser);
PASS();
}
SUITE(mem) {
/* mem API */
RUN_TEST(mem_arena_eager_commit_follows_platform_commit_cost);
@@ -1350,4 +1399,7 @@ SUITE(mem) {
RUN_TEST(parallel_extract_without_source_retention);
RUN_TEST(parallel_extract_tiny_source_retention_budget);
RUN_TEST(parallel_extract_with_slab);
/* extraction scratch arena (#2010) */
RUN_TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010);
}