diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h
index fb5e446a5..c0bba7792 100644
--- a/internal/cbm/cbm.h
+++ b/internal/cbm/cbm.h
@@ -606,6 +606,11 @@ typedef struct CBMFileResult {
// by cbm_free_result(); ordinary single-file results leave these zeroed.
struct CBMFileResult **owned_results;
int owned_result_count;
+
+ /* The file's own doc, set on its File node: the Go package comment or
+ * the Rust inner docs (//!). NULL for other languages and undocumented
+ * files. */
+ const char *module_doc;
} CBMFileResult;
// --- Enclosing function cache ---
@@ -692,6 +697,11 @@ typedef struct {
/* How many nodes the unified walk actually visited (whether or not it ran
* out of budget) — the measurement the budget has to be expressed in. */
uint32_t walk_nodes_visited;
+ /* Doc-comment lookup state (extract_defs.c), NULL until first used and
+ * allocated in `scratch`: the memo of parents' child arrays and the Perl
+ * POD section index. */
+ void *doc_memo;
+ void *doc_pod_index;
} CBMExtractCtx;
// --- Public API ---
diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c
index bb56e2870..b3b6e6acc 100644
--- a/internal/cbm/extract_defs.c
+++ b/internal/cbm/extract_defs.c
@@ -1265,36 +1265,6 @@ static bool is_comment_node(const char *kind) {
strcmp(kind, "line_comment") == 0 || strcmp(kind, "multiline_comment") == 0);
}
-// Extract comment text, truncating to MAX_COMMENT_LEN.
-// #1017: snap the cut point back to a complete UTF-8 codepoint boundary.
-static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) {
- char *text = cbm_node_text(a, node, source);
- if (text && strlen(text) > MAX_COMMENT_LEN) {
- size_t cut = MAX_COMMENT_LEN;
- while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80)
- cut--;
- text[cut] = '\0';
- }
- return text;
-}
-
-// Go-specific: type_spec/type_alias comment is before the parent type_declaration.
-static const char *extract_go_type_docstring(CBMArena *a, TSNode node, const char *source) {
- const char *kind = ts_node_type(node);
- if (strcmp(kind, "type_spec") != 0 && strcmp(kind, "type_alias") != 0) {
- return NULL;
- }
- TSNode parent = ts_node_parent(node);
- if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "type_declaration") != 0) {
- return NULL;
- }
- TSNode pprev = ts_node_prev_sibling(parent);
- if (!ts_node_is_null(pprev) && is_comment_node(ts_node_type(pprev))) {
- return extract_comment_text(a, pprev, source);
- }
- return NULL;
-}
-
// Python-specific: docstring as first expression_statement -> string in function body.
static const char *extract_python_docstring(CBMArena *a, TSNode node, const char *source) {
TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body"));
@@ -1314,30 +1284,1423 @@ static const char *extract_python_docstring(CBMArena *a, TSNode node, const char
}
const char *sk = ts_node_type(str);
if (strcmp(sk, "string") == 0 || strcmp(sk, "concatenated_string") == 0) {
- return extract_comment_text(a, str, source);
+ return cbm_node_text(a, str, source);
}
return NULL;
}
-// Extract docstring from the node's leading comment.
-static const char *extract_docstring(CBMArena *a, TSNode node, const char *source,
- CBMLanguage lang) {
- if (lang == CBM_LANG_GO) {
- const char *doc = extract_go_type_docstring(a, node, source);
- if (doc) {
- return doc;
+/* ── Doc comments ─────────────────────────────────────────────────────────
+ * A definition's doc is the comment RUN directly above it, not only the one
+ * comment node before it:
+ * - a run is contiguous (no blank line) and of one marker style; a Go comment
+ * group may mix styles and drops go/ast directives (//go:generate, //line);
+ * - it sits above the item's attributes (Rust), its declaration or export
+ * wrapper (JS/TS, C#, Go var/const) or its typedef (C);
+ * - a comment on the previous code's last line is that code's trailing
+ * comment, and a Rust inner doc documents the module, never the item;
+ * - a nearer plain comment does not hide a doc-style comment directly above;
+ * - a run a blank line away is detached (Go: always; elsewhere: unless it is
+ * doc-style), and a run with no letter or digit (a banner) is no doc;
+ * - the whole run is kept: the node's properties buffer grows with the doc.
+ * Fields, variables, enum members and macros of code languages get the same
+ * doc, a Perl sub the POD section that names it, and a Go or Rust file its
+ * package comment or inner docs (on the File node).
+ *
+ * Leading trivia is read from the parent's child array, kept in a small
+ * per-file memo: tree-sitter has no parent pointers, so ts_node_parent and
+ * ts_node_prev_sibling are O(position), and one lookup per member of an
+ * N-member class body cost O(N^2) (65,536 fields in one dotnet test class). */
+
+enum {
+ DOC_SPAN_COMMENT = 1,
+ DOC_SPAN_ATTR = 2,
+ DOC_SPAN_INIT_CAP = 8,
+ DOC_SPAN_GROW = 2,
+ DOC_MEMO_SLOTS = 8, /* parents whose child arrays stay cached */
+ DOC_MEMO_WIDE = 32, /* a level wider than this is cached while descending */
+ DOC_BISECT = 2,
+};
+
+/* Comment marker families (the slash forms are spelled out in words so this
+ * comment does not nest). */
+typedef enum {
+ DOC_CS_OTHER = 0, /* another family (--, ;, %) */
+ DOC_CS_LINE, /* slash-slash, or four and more slashes */
+ DOC_CS_LINE_DOC, /* slash-slash-slash */
+ DOC_CS_LINE_BANG, /* slash-slash-bang */
+ DOC_CS_BLOCK, /* plain block, empty block, star-banner block */
+ DOC_CS_BLOCK_DOC, /* slash-star-star */
+ DOC_CS_BLOCK_BANG, /* slash-star-bang */
+ DOC_CS_HASH, /* hash */
+} doc_style_t;
+
+/* One trivia element as a byte span. erow is the EFFECTIVE end row: a node
+ * that ends at column 0 (a Rust line_comment includes its newline) ends on the
+ * row before. */
+typedef struct {
+ uint32_t sb;
+ uint32_t eb;
+ uint32_t srow;
+ uint32_t erow;
+ uint8_t kind;
+ uint8_t style;
+} doc_span_t;
+
+typedef struct {
+ doc_span_t *items; /* trivia directly before the anchor, in source order */
+ int count;
+ int cap;
+ bool code_before; /* a non-trivia sibling precedes items[0] */
+ uint32_t code_erow; /* ... its effective end row */
+ uint32_t code_eb; /* ... its end byte (Kotlin gap scan) */
+ uint32_t code_end_row; /* ... its raw end row (Kotlin gap scan) */
+} doc_trivia_t;
+
+static CBMArena *doc_scratch(CBMExtractCtx *ctx) {
+ return ctx->scratch ? ctx->scratch : ctx->arena;
+}
+
+static uint32_t doc_erow(TSNode n) {
+ TSPoint s = ts_node_start_point(n);
+ TSPoint e = ts_node_end_point(n);
+ return (e.column == 0 && e.row > s.row) ? e.row - SKIP_ONE : e.row;
+}
+
+static bool doc_has_prefix(const char *p, uint32_t n, const char *lit) {
+ size_t len = strlen(lit);
+ return n >= len && memcmp(p, lit, len) == 0;
+}
+
+static doc_style_t doc_style_of(const char *src, uint32_t sb, uint32_t eb) {
+ const char *p = src + sb;
+ uint32_t n = eb > sb ? eb - sb : 0;
+ if (doc_has_prefix(p, n, "////")) {
+ return DOC_CS_LINE;
+ }
+ if (doc_has_prefix(p, n, "///")) {
+ return DOC_CS_LINE_DOC;
+ }
+ if (doc_has_prefix(p, n, "//!")) {
+ return DOC_CS_LINE_BANG;
+ }
+ if (doc_has_prefix(p, n, "//")) {
+ return DOC_CS_LINE;
+ }
+ if (doc_has_prefix(p, n, "/**/") || doc_has_prefix(p, n, "/***")) {
+ return DOC_CS_BLOCK;
+ }
+ if (doc_has_prefix(p, n, "/**")) {
+ return DOC_CS_BLOCK_DOC;
+ }
+ if (doc_has_prefix(p, n, "/*!")) {
+ return DOC_CS_BLOCK_BANG;
+ }
+ if (doc_has_prefix(p, n, "/*")) {
+ return DOC_CS_BLOCK;
+ }
+ return doc_has_prefix(p, n, "#") ? DOC_CS_HASH : DOC_CS_OTHER;
+}
+
+/* Languages whose toolchain has a doc-comment syntax distinct from a plain
+ * comment (javadoc / KDoc / JSDoc / PHPDoc, rustdoc, C# XML docs, Doxygen). */
+static bool doc_lang_has_doc_syntax(CBMLanguage lang) {
+ switch (lang) {
+ case CBM_LANG_RUST:
+ case CBM_LANG_C:
+ case CBM_LANG_CPP:
+ case CBM_LANG_CUDA:
+ case CBM_LANG_OBJC:
+ case CBM_LANG_CSHARP:
+ case CBM_LANG_SWIFT:
+ case CBM_LANG_DART:
+ case CBM_LANG_JAVA:
+ case CBM_LANG_KOTLIN:
+ case CBM_LANG_SCALA:
+ case CBM_LANG_GROOVY:
+ case CBM_LANG_JAVASCRIPT:
+ case CBM_LANG_TYPESCRIPT:
+ case CBM_LANG_TSX:
+ case CBM_LANG_ARKTS:
+ case CBM_LANG_PHP:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool doc_style_is_doc(CBMLanguage lang, int style) {
+ switch (lang) {
+ case CBM_LANG_C:
+ case CBM_LANG_CPP:
+ case CBM_LANG_CUDA:
+ case CBM_LANG_OBJC:
+ return style == DOC_CS_LINE_DOC || style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_DOC ||
+ style == DOC_CS_BLOCK_BANG;
+ case CBM_LANG_RUST:
+ case CBM_LANG_CSHARP:
+ case CBM_LANG_SWIFT:
+ case CBM_LANG_DART:
+ return style == DOC_CS_LINE_DOC || style == DOC_CS_BLOCK_DOC;
+ default:
+ return doc_lang_has_doc_syntax(lang) && style == DOC_CS_BLOCK_DOC;
+ }
+}
+
+/* Field/Variable docs are for code languages: config languages (YAML, TOML,
+ * ...) mint a Variable per key, and a comment there is no API doc. */
+static bool doc_lang_member_docs(CBMLanguage lang) {
+ return doc_lang_has_doc_syntax(lang) || lang == CBM_LANG_GO || lang == CBM_LANG_PERL;
+}
+
+static uint8_t doc_trivia_kind(CBMLanguage lang, const char *kind) {
+ if (is_comment_node(kind)) {
+ return DOC_SPAN_COMMENT;
+ }
+ /* A Rust #[attr] is a sibling between the doc and the item. */
+ if (lang == CBM_LANG_RUST && strcmp(kind, "attribute_item") == 0) {
+ return DOC_SPAN_ATTR;
+ }
+ return 0;
+}
+
+static doc_span_t doc_span_of(TSNode n, const char *src, uint8_t kind) {
+ doc_span_t sp;
+ sp.sb = ts_node_start_byte(n);
+ sp.eb = ts_node_end_byte(n);
+ sp.srow = ts_node_start_point(n).row;
+ sp.erow = doc_erow(n);
+ sp.kind = kind;
+ sp.style = (uint8_t)(kind == DOC_SPAN_COMMENT ? doc_style_of(src, sp.sb, sp.eb) : DOC_CS_OTHER);
+ return sp;
+}
+
+static void doc_push_span(CBMArena *a, doc_trivia_t *t, const doc_span_t *sp) {
+ if (t->count == t->cap) {
+ int ncap = t->cap ? t->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP;
+ doc_span_t *grown = (doc_span_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(doc_span_t));
+ if (!grown) {
+ return;
+ }
+ if (t->count > 0) {
+ memcpy(grown, t->items, (size_t)t->count * sizeof(doc_span_t));
+ }
+ t->items = grown;
+ t->cap = ncap;
+ }
+ t->items[t->count++] = *sp;
+}
+
+/* A non-trivia sibling before the anchor: trivia seen so far are not its. */
+static void doc_note_code(doc_trivia_t *t, TSNode code) {
+ t->count = 0;
+ t->code_before = true;
+ t->code_erow = doc_erow(code);
+ t->code_eb = ts_node_end_byte(code);
+ t->code_end_row = ts_node_end_point(code).row;
+}
+
+/* End of the Kotlin block comment opening at src[i] (block comments nest). */
+static uint32_t doc_kotlin_block_end(const char *src, uint32_t i, uint32_t to, uint32_t *row) {
+ int depth = 0;
+ while (i < to) {
+ bool pair = i + SKIP_ONE < to;
+ if (pair && src[i] == '/' && src[i + SKIP_ONE] == '*') {
+ depth++;
+ i += PAIR_LEN;
+ continue;
+ }
+ if (pair && src[i] == '*' && src[i + SKIP_ONE] == '/') {
+ depth--;
+ i += PAIR_LEN;
+ if (depth == 0) {
+ return i;
+ }
+ continue;
+ }
+ if (src[i] == '\n') {
+ (*row)++;
+ }
+ i++;
+ }
+ return i;
+}
+
+/* tree-sitter-kotlin's automatic-semicolon scanner can swallow a comment that
+ * sits between two declarations (seen before `enum class` and `abstract
+ * class`): it has no node and survives only as bytes in the gap between the
+ * siblings. Recover the comments from that gap. Any other byte is a token the
+ * tree does not show, and the run restarts after it. */
+static void doc_scan_kotlin_gap(CBMArena *a, const char *src, uint32_t from, uint32_t to,
+ uint32_t row, doc_trivia_t *t) {
+ uint32_t i = from;
+ while (i < to) {
+ char c = src[i];
+ if (c == '\n') {
+ row++;
+ i++;
+ continue;
+ }
+ if (c == ' ' || c == '\t' || c == '\r' || c == '\f') {
+ i++;
+ continue;
+ }
+ if (c == '/' && i + SKIP_ONE < to &&
+ (src[i + SKIP_ONE] == '/' || src[i + SKIP_ONE] == '*')) {
+ doc_span_t sp = {i, i, row, row, DOC_SPAN_COMMENT, DOC_CS_OTHER};
+ if (src[i + SKIP_ONE] == '/') {
+ while (i < to && src[i] != '\n') {
+ i++;
+ }
+ } else {
+ i = doc_kotlin_block_end(src, i, to, &row);
+ }
+ sp.eb = i;
+ sp.erow = row;
+ sp.style = (uint8_t)doc_style_of(src, sp.sb, sp.eb);
+ doc_push_span(a, t, &sp);
+ continue;
+ }
+ t->count = 0;
+ t->code_before = true;
+ t->code_erow = row;
+ i++;
+ }
+}
+
+static bool doc_space_byte(char c) {
+ return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f';
+}
+
+/* Start of the (nesting) block comment whose close ends at src[p), or
+ * UINT32_MAX; *row moves to its start row. */
+static uint32_t doc_back_block_start(const char *src, uint32_t p, uint32_t *row) {
+ uint32_t q = p;
+ int depth = 0;
+ while (q >= PAIR_LEN) {
+ if (src[q - PAIR_LEN] == '*' && src[q - SKIP_ONE] == '/') {
+ depth++;
+ q -= PAIR_LEN;
+ continue;
+ }
+ if (src[q - PAIR_LEN] == '/' && src[q - SKIP_ONE] == '*') {
+ depth--;
+ q -= PAIR_LEN;
+ if (depth == 0) {
+ return q;
+ }
+ continue;
+ }
+ if (src[q - SKIP_ONE] == '\n' && *row > 0) {
+ (*row)--;
+ }
+ q--;
+ }
+ return UINT32_MAX;
+}
+
+/* Start of a line comment that fills its line up to src[p), or UINT32_MAX. */
+static uint32_t doc_back_line_start(const char *src, uint32_t p) {
+ uint32_t ls = p;
+ while (ls > 0 && src[ls - SKIP_ONE] != '\n') {
+ ls--;
+ }
+ uint32_t k = ls;
+ while (k < p && (src[k] == ' ' || src[k] == '\t')) {
+ k++;
+ }
+ return (k + SKIP_ONE < p && src[k] == '/' && src[k + SKIP_ONE] == '/') ? k : UINT32_MAX;
+}
+
+/* tree-sitter-kotlin can also fold such a comment into the PREVIOUS node's
+ * byte range (package_header ends after the comment that follows it), so
+ * neither a sibling nor the gap holds it. Scan backwards from the anchor over
+ * whitespace, block comments and whole-line line comments; stop at anything
+ * else. The spans come out nearest-first and are appended in source order. */
+static void doc_scan_kotlin_back(CBMArena *a, const char *src, uint32_t anchor_sb,
+ uint32_t anchor_row, doc_trivia_t *t) {
+ doc_trivia_t rev;
+ memset(&rev, 0, sizeof(rev));
+ uint32_t p = anchor_sb;
+ uint32_t row = anchor_row;
+ for (;;) {
+ while (p > 0 && doc_space_byte(src[p - SKIP_ONE])) {
+ if (src[p - SKIP_ONE] == '\n' && row > 0) {
+ row--;
+ }
+ p--;
+ }
+ uint32_t srow = row;
+ uint32_t start = (p >= PAIR_LEN && src[p - PAIR_LEN] == '*' && src[p - SKIP_ONE] == '/')
+ ? doc_back_block_start(src, p, &srow)
+ : doc_back_line_start(src, p);
+ if (start == UINT32_MAX) {
+ break;
+ }
+ doc_span_t sp = {
+ start, p, srow, row, DOC_SPAN_COMMENT, (uint8_t)doc_style_of(src, start, p)};
+ doc_push_span(a, &rev, &sp);
+ p = start;
+ row = srow;
+ }
+ for (int i = rev.count - SKIP_ONE; i >= 0; i--) {
+ doc_push_span(a, t, &rev.items[i]);
+ }
+ t->code_before = p > 0;
+ t->code_erow = row;
+}
+
+/* The memo: the child arrays of the last few parents looked up. A node's index
+ * is a binary search on start bytes, and its parent is found by descending
+ * from the deepest cached ancestor (the root is always reachable): the same
+ * answers as the tree-sitter calls, O(log N) per lookup on wide levels. */
+typedef struct {
+ TSNode node;
+ TSNode *kids;
+ uint32_t *kid_sb;
+ uint32_t n;
+ uint32_t cap;
+ uint32_t stamp;
+ bool used;
+} doc_memo_ent_t;
+
+typedef struct {
+ doc_memo_ent_t ent[DOC_MEMO_SLOTS];
+ uint32_t clock;
+} doc_memo_t;
+
+/* The cached entry of `parent`, or the slot to reuse for it (unused first,
+ * else least recently used). */
+static doc_memo_ent_t *doc_memo_slot(doc_memo_t *m, TSNode parent, bool *hit) {
+ doc_memo_ent_t *victim = &m->ent[0];
+ for (int i = 0; i < DOC_MEMO_SLOTS; i++) {
+ doc_memo_ent_t *e = &m->ent[i];
+ if (e->used && ts_node_eq(e->node, parent)) {
+ *hit = true;
+ return e;
+ }
+ if (victim->used && (!e->used || e->stamp < victim->stamp)) {
+ victim = e;
}
}
+ *hit = false;
+ return victim;
+}
- TSNode prev = ts_node_prev_sibling(node);
- if (!ts_node_is_null(prev) && is_comment_node(ts_node_type(prev))) {
- return extract_comment_text(a, prev, source);
+/* Fill `e` with the children of `parent`; false when the arrays cannot grow. */
+static bool doc_memo_fill(CBMArena *a, doc_memo_ent_t *e, TSNode parent) {
+ uint32_t n = ts_node_child_count(parent);
+ if (n > e->cap) {
+ TSNode *kids = (TSNode *)cbm_arena_alloc(a, (size_t)n * sizeof(TSNode));
+ uint32_t *sb = (uint32_t *)cbm_arena_alloc(a, (size_t)n * sizeof(uint32_t));
+ if (!kids || !sb) {
+ e->used = false;
+ return false;
+ }
+ e->kids = kids;
+ e->kid_sb = sb;
+ e->cap = n;
}
+ uint32_t k = 0;
+ TSTreeCursor cur = ts_tree_cursor_new(parent);
+ if (ts_tree_cursor_goto_first_child(&cur)) {
+ do {
+ e->kids[k] = ts_tree_cursor_current_node(&cur);
+ e->kid_sb[k] = ts_node_start_byte(e->kids[k]);
+ k++;
+ } while (k < n && ts_tree_cursor_goto_next_sibling(&cur));
+ }
+ ts_tree_cursor_delete(&cur);
+ e->node = parent;
+ e->n = k;
+ e->used = true;
+ return true;
+}
- if (lang == CBM_LANG_PYTHON) {
- return extract_python_docstring(a, node, source);
+static const doc_memo_ent_t *doc_memo_get(CBMExtractCtx *ctx, TSNode parent) {
+ CBMArena *a = doc_scratch(ctx);
+ if (!ctx->doc_memo) {
+ doc_memo_t *fresh = (doc_memo_t *)cbm_arena_alloc(a, sizeof(doc_memo_t));
+ if (!fresh) {
+ return NULL;
+ }
+ memset(fresh, 0, sizeof(*fresh));
+ ctx->doc_memo = fresh;
}
- return NULL;
+ doc_memo_t *m = (doc_memo_t *)ctx->doc_memo;
+ m->clock++;
+ bool hit = false;
+ doc_memo_ent_t *e = doc_memo_slot(m, parent, &hit);
+ if (!hit && !doc_memo_fill(a, e, parent)) {
+ return NULL;
+ }
+ e->stamp = m->clock;
+ return e;
+}
+
+/* Index of `node` among the cached children, or -1. */
+static int doc_memo_index(const doc_memo_ent_t *e, TSNode node) {
+ uint32_t s = ts_node_start_byte(node);
+ uint32_t lo = 0;
+ uint32_t hi = e->n;
+ while (lo < hi) {
+ uint32_t mid = lo + ((hi - lo) / DOC_BISECT);
+ if (e->kid_sb[mid] < s) {
+ lo = mid + SKIP_ONE;
+ } else {
+ hi = mid;
+ }
+ }
+ for (uint32_t i = lo; i < e->n && e->kid_sb[i] == s; i++) {
+ if (ts_node_eq(e->kids[i], node)) {
+ return (int)i;
+ }
+ }
+ return CBM_NOT_FOUND;
+}
+
+/* Index of the cached child whose byte range contains `node`, or -1. */
+static int doc_memo_containing(const doc_memo_ent_t *e, TSNode node) {
+ uint32_t s = ts_node_start_byte(node);
+ uint32_t en = ts_node_end_byte(node);
+ uint32_t lo = 0;
+ uint32_t hi = e->n;
+ while (lo < hi) {
+ uint32_t mid = lo + ((hi - lo) / DOC_BISECT);
+ if (e->kid_sb[mid] <= s) {
+ lo = mid + SKIP_ONE;
+ } else {
+ hi = mid;
+ }
+ }
+ for (int i = (int)lo - SKIP_ONE; i >= 0; i--) {
+ TSNode kid = e->kids[i];
+ uint32_t ke = ts_node_end_byte(kid);
+ if (ke >= en && (ke > e->kid_sb[i] || ts_node_eq(kid, node))) {
+ return i;
+ }
+ if (ke <= s && e->kid_sb[i] < s) {
+ break;
+ }
+ }
+ return CBM_NOT_FOUND;
+}
+
+/* The deepest cached node that contains `node` (the node itself excluded). */
+static const doc_memo_ent_t *doc_memo_ancestor(const doc_memo_t *m, TSNode node) {
+ const doc_memo_ent_t *best = NULL;
+ uint32_t ns = ts_node_start_byte(node);
+ uint32_t ne = ts_node_end_byte(node);
+ for (int i = 0; m && i < DOC_MEMO_SLOTS; i++) {
+ const doc_memo_ent_t *e = &m->ent[i];
+ if (!e->used || ts_node_eq(e->node, node)) {
+ continue;
+ }
+ uint32_t es = ts_node_start_byte(e->node);
+ uint32_t ee = ts_node_end_byte(e->node);
+ if (es <= ns && ne <= ee &&
+ (!best || ee - es < ts_node_end_byte(best->node) - ts_node_start_byte(best->node))) {
+ best = e;
+ }
+ }
+ return best;
+}
+
+/* ts_node_parent() through the memo: descend from the deepest cached ancestor,
+ * binary search on wide levels (cached on the way), linear on narrow ones. */
+static TSNode doc_parent(CBMExtractCtx *ctx, TSNode node) {
+ if (ts_node_eq(node, ctx->root)) {
+ TSNode none = {0};
+ return none;
+ }
+ const doc_memo_ent_t *e = doc_memo_ancestor((const doc_memo_t *)ctx->doc_memo, node);
+ if (!e) {
+ e = doc_memo_get(ctx, ctx->root);
+ }
+ while (e) {
+ int k = doc_memo_containing(e, node);
+ if (k < 0) {
+ break;
+ }
+ TSNode cur = e->kids[k];
+ if (ts_node_eq(cur, node)) {
+ return e->node;
+ }
+ e = NULL;
+ while (!ts_node_is_null(cur)) {
+ if (ts_node_child_count(cur) > DOC_MEMO_WIDE) {
+ e = doc_memo_get(ctx, cur);
+ break;
+ }
+ TSNode next = ts_node_child_with_descendant(cur, node);
+ if (ts_node_is_null(next)) {
+ break;
+ }
+ if (ts_node_eq(next, node)) {
+ return cur;
+ }
+ cur = next;
+ }
+ }
+ return ts_node_parent(node); /* not reachable through the memo */
+}
+
+/* Trivia before the child at index mk of a cached child array. */
+static void doc_collect_cached(CBMExtractCtx *ctx, const doc_memo_ent_t *me, int mk,
+ doc_trivia_t *t) {
+ int j = mk - SKIP_ONE;
+ while (j >= 0 && doc_trivia_kind(ctx->language, ts_node_type(me->kids[j]))) {
+ j--;
+ }
+ if (j >= 0) {
+ doc_note_code(t, me->kids[j]);
+ }
+ for (int i = j + SKIP_ONE; i < mk; i++) {
+ uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(me->kids[i]));
+ doc_span_t sp = doc_span_of(me->kids[i], ctx->source, kind);
+ doc_push_span(doc_scratch(ctx), t, &sp);
+ }
+}
+
+/* The same without the memo (its arrays could not be allocated): one cursor
+ * pass over the parent. False when `anchor` is not among its children. */
+static bool doc_collect_cursor(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) {
+ bool found = false;
+ TSTreeCursor cur = ts_tree_cursor_new(parent);
+ if (ts_tree_cursor_goto_first_child(&cur)) {
+ do {
+ TSNode ch = ts_tree_cursor_current_node(&cur);
+ if (ts_node_eq(ch, anchor)) {
+ found = true;
+ break;
+ }
+ uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(ch));
+ if (kind) {
+ doc_span_t sp = doc_span_of(ch, ctx->source, kind);
+ doc_push_span(doc_scratch(ctx), t, &sp);
+ } else {
+ doc_note_code(t, ch);
+ }
+ } while (ts_tree_cursor_goto_next_sibling(&cur));
+ }
+ ts_tree_cursor_delete(&cur);
+ return found;
+}
+
+/* Kotlin comments the grammar hid: in the gap after the previous sibling, else
+ * folded into the previous node (see the two scanners above). */
+static void doc_collect_kotlin(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) {
+ CBMArena *a = doc_scratch(ctx);
+ uint32_t from = t->code_before ? t->code_eb : ts_node_start_byte(parent);
+ uint32_t row = t->code_before ? t->code_end_row : ts_node_start_point(parent).row;
+ uint32_t to = ts_node_start_byte(anchor);
+ if (from < to) {
+ doc_scan_kotlin_gap(a, ctx->source, from, to, row, t);
+ }
+ if (t->count == 0) {
+ doc_scan_kotlin_back(a, ctx->source, to, ts_node_start_point(anchor).row, t);
+ }
+}
+
+/* Leading trivia of `anchor`, in source order. */
+static void doc_collect_trivia(CBMExtractCtx *ctx, TSNode anchor, doc_trivia_t *t) {
+ memset(t, 0, sizeof(*t));
+ TSNode parent = doc_parent(ctx, anchor);
+ if (ts_node_is_null(parent)) {
+ return;
+ }
+ const doc_memo_ent_t *me = doc_memo_get(ctx, parent);
+ int mk = me ? doc_memo_index(me, anchor) : CBM_NOT_FOUND;
+ bool found = mk >= 0;
+ if (found) {
+ doc_collect_cached(ctx, me, mk, t);
+ } else {
+ found = doc_collect_cursor(ctx, parent, anchor, t);
+ }
+ if (!found) {
+ memset(t, 0, sizeof(*t));
+ return;
+ }
+ if (t->count == 0 && ctx->language == CBM_LANG_KOTLIN) {
+ doc_collect_kotlin(ctx, parent, anchor, t);
+ }
+}
+
+/* go/ast CommentGroup.Text: a line comment with no space after the slashes is
+ * a directive, not documentation: //line, //extern, //export and the
+ * //[a-z0-9]+:[a-z0-9] form (//go:generate, //nolint:errcheck, ...). */
+static bool doc_go_is_directive(const char *src, const doc_span_t *sp) {
+ const char *p = src + sp->sb;
+ uint32_t n = sp->eb > sp->sb ? sp->eb - sp->sb : 0;
+ if (n <= PAIR_LEN || p[0] != '/' || p[SKIP_ONE] != '/' || p[PAIR_LEN] == ' ') {
+ return false;
+ }
+ p += PAIR_LEN;
+ n -= PAIR_LEN;
+ if (doc_has_prefix(p, n, "line ") || doc_has_prefix(p, n, "extern ") ||
+ doc_has_prefix(p, n, "export ")) {
+ return true;
+ }
+ uint32_t colon = 0;
+ while (colon < n && p[colon] != ':' && p[colon] != '\n') {
+ colon++;
+ }
+ if (colon == 0 || colon >= n || p[colon] != ':' || colon + SKIP_ONE >= n) {
+ return false;
+ }
+ for (uint32_t i = 0; i <= colon + SKIP_ONE; i++) {
+ char b = p[i];
+ if (i != colon && !((b >= 'a' && b <= 'z') || (b >= '0' && b <= '9'))) {
+ return false;
+ }
+ }
+ return true;
+}
+
+/* UTF-8 bytes of the banner characters (see doc_has_words). */
+enum {
+ DOC_U8_ASCII_END = 0x80,
+ DOC_U8_CONT_MASK = 0xC0, /* 10xxxxxx: continuation byte */
+ DOC_U8_CONT = 0x80,
+ DOC_U8_LATIN1 = 0xC2, /* lead byte of U+0080..U+00BF */
+ DOC_U8_MIDDLE_DOT = 0xB7, /* U+00B7 */
+ DOC_U8_U2000 = 0xE2, /* lead byte of U+2000..U+2FFF */
+ DOC_U8_PUNCT = 0x80, /* U+2000..U+203F */
+ DOC_U8_DASH_FIRST = 0x90, /* U+2010 hyphen */
+ DOC_U8_BULLET_LAST = 0xA7, /* U+2027 hyphenation point */
+ DOC_U8_BOX_FIRST = 0x94, /* U+2500 box drawing */
+ DOC_U8_SHAPES_LAST = 0x97, /* U+25FF, end of the geometric shapes */
+ DOC_U8_LATIN1_LEN = 2,
+ DOC_U8_U2000_LEN = 3,
+};
+
+static bool doc_banner_char(const unsigned char *p, size_t left) {
+ if (left >= DOC_U8_LATIN1_LEN && p[0] == DOC_U8_LATIN1 && p[SKIP_ONE] == DOC_U8_MIDDLE_DOT) {
+ return true;
+ }
+ if (left < DOC_U8_U2000_LEN || p[0] != DOC_U8_U2000) {
+ return false;
+ }
+ if (p[SKIP_ONE] >= DOC_U8_BOX_FIRST && p[SKIP_ONE] <= DOC_U8_SHAPES_LAST) {
+ return true;
+ }
+ return p[SKIP_ONE] == DOC_U8_PUNCT && p[PAIR_LEN] >= DOC_U8_DASH_FIRST &&
+ p[PAIR_LEN] <= DOC_U8_BULLET_LAST;
+}
+
+/* A comment with no letter or digit is a divider or banner, not documentation.
+ * Any non-ASCII character counts as a letter except the ones banners are drawn
+ * with: U+00B7, the dashes and bullets U+2010..U+2027, and the box-drawing,
+ * block and geometric-shape characters U+2500..U+25FF. */
+static bool doc_has_words(const char *s, size_t n) {
+ const unsigned char *u = (const unsigned char *)s;
+ for (size_t i = 0; i < n; i++) {
+ if (u[i] < DOC_U8_ASCII_END) {
+ if (isalnum(u[i])) {
+ return true;
+ }
+ } else if ((u[i] & DOC_U8_CONT_MASK) != DOC_U8_CONT && !doc_banner_char(u + i, n - i)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool doc_span_kept(const char *src, const doc_span_t *sp, bool go_directives) {
+ return sp->kind == DOC_SPAN_COMMENT && !(go_directives && doc_go_is_directive(src, sp));
+}
+
+/* Text of items[first..last]: one comment keeps its exact bytes; several are
+ * joined by newlines, each without its own trailing newline. NULL when nothing
+ * is left (Go directives only) or when no comment has a letter or digit. */
+static const char *doc_run_text(CBMExtractCtx *ctx, const doc_trivia_t *t, int first, int last,
+ bool go_directives) {
+ const char *src = ctx->source;
+ int kept = 0;
+ bool words = false;
+ size_t total = 0;
+ for (int k = first; k <= last; k++) {
+ const doc_span_t *sp = &t->items[k];
+ if (!doc_span_kept(src, sp, go_directives)) {
+ continue;
+ }
+ total += (size_t)(sp->eb - sp->sb) + SKIP_ONE;
+ words = words || doc_has_words(src + sp->sb, sp->eb - sp->sb);
+ kept++;
+ }
+ if (kept == 0 || !words) {
+ return NULL;
+ }
+ char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE);
+ if (!buf) {
+ return NULL;
+ }
+ size_t w = 0;
+ for (int k = first; k <= last; k++) {
+ const doc_span_t *sp = &t->items[k];
+ if (!doc_span_kept(src, sp, go_directives)) {
+ continue;
+ }
+ uint32_t eb = sp->eb;
+ while (kept > SKIP_ONE && eb > sp->sb &&
+ (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r')) {
+ eb--;
+ }
+ if (w > 0) {
+ buf[w++] = '\n';
+ }
+ memcpy(buf + w, src + sp->sb, eb - sp->sb);
+ w += eb - sp->sb;
+ }
+ buf[w] = '\0';
+ return buf;
+}
+
+/* Index of the comment nearest the item, above its Rust attributes, or -1. */
+static int doc_nearest_comment(const doc_trivia_t *t) {
+ int k = t->count - SKIP_ONE;
+ while (k >= 0 && t->items[k].kind == DOC_SPAN_ATTR) {
+ k--;
+ }
+ return (k >= 0 && t->items[k].kind == DOC_SPAN_COMMENT) ? k : CBM_NOT_FOUND;
+}
+
+/* A nearer plain comment does not hide a doc-style comment directly above it
+ * (a javadoc, then `// NOSONAR`): the index of that doc comment, else `near`. */
+static int doc_prefer_doc_style(CBMLanguage lang, const doc_trivia_t *t, int near) {
+ if (!doc_lang_has_doc_syntax(lang) || doc_style_is_doc(lang, t->items[near].style)) {
+ return near;
+ }
+ for (int j = near - SKIP_ONE; j >= 0; j--) {
+ const doc_span_t *sp = &t->items[j];
+ if (t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) {
+ break; /* a blank line */
+ }
+ if (sp->kind == DOC_SPAN_ATTR) {
+ continue;
+ }
+ if (sp->kind != DOC_SPAN_COMMENT) {
+ break;
+ }
+ if (doc_style_is_doc(lang, sp->style)) {
+ return j;
+ }
+ }
+ return near;
+}
+
+/* First index of the run that ends at `near`: contiguous, one style (a Go
+ * comment group mixes styles), not the previous code's trailing comment. */
+static int doc_run_first(CBMLanguage lang, const doc_trivia_t *t, int near) {
+ int style = t->items[near].style;
+ int first = near;
+ for (int j = near - SKIP_ONE; j >= 0; j--) {
+ const doc_span_t *sp = &t->items[j];
+ if (sp->kind != DOC_SPAN_COMMENT || t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) {
+ break; /* an attribute or a blank line ends the run */
+ }
+ if (lang != CBM_LANG_GO && sp->style != style) {
+ break;
+ }
+ if (j == 0 && t->code_before && sp->srow == t->code_erow) {
+ break;
+ }
+ first = j;
+ }
+ return first;
+}
+
+/* The doc of an anchor from its leading trivia. */
+static const char *doc_from_trivia(CBMExtractCtx *ctx, const doc_trivia_t *t,
+ uint32_t anchor_srow) {
+ CBMLanguage lang = ctx->language;
+ int near = t->items ? doc_nearest_comment(t) : CBM_NOT_FOUND;
+ if (near < 0) {
+ return NULL;
+ }
+ near = doc_prefer_doc_style(lang, t, near);
+ int style = t->items[near].style;
+ if (lang == CBM_LANG_RUST && (style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_BANG)) {
+ return NULL; /* an inner doc: it documents the enclosing module */
+ }
+ if (near == 0 && t->code_before && t->items[0].srow == t->code_erow) {
+ return NULL; /* the previous code's trailing comment */
+ }
+ uint32_t next_row = near + SKIP_ONE < t->count ? t->items[near + SKIP_ONE].srow : anchor_srow;
+ bool detached = next_row > t->items[near].erow + SKIP_ONE;
+ if (detached && (lang == CBM_LANG_GO || !doc_style_is_doc(lang, style))) {
+ return NULL;
+ }
+ return doc_run_text(ctx, t, doc_run_first(lang, t, near), near, lang == CBM_LANG_GO);
+}
+
+static const char *doc_for_anchor(CBMExtractCtx *ctx, TSNode anchor) {
+ doc_trivia_t t;
+ doc_collect_trivia(ctx, anchor, &t);
+ return doc_from_trivia(ctx, &t, ts_node_start_point(anchor).row);
+}
+
+static bool doc_kind_is(TSNode n, const char *kind) {
+ return !ts_node_is_null(n) && strcmp(ts_node_type(n), kind) == 0;
+}
+
+/* JS/TS: a function or class expression is documented above its declarator,
+ * field, pair or assignment statement, a declarator above its declaration,
+ * and any of them above `declare` and `export`. */
+static TSNode doc_anchor_js(CBMExtractCtx *ctx, TSNode cur) {
+ const char *k = ts_node_type(cur);
+ if (strcmp(k, "arrow_function") == 0 || strcmp(k, "function_expression") == 0 ||
+ strcmp(k, "generator_function") == 0 || strcmp(k, "class") == 0) {
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "variable_declarator") || doc_kind_is(p, "public_field_definition") ||
+ doc_kind_is(p, "field_definition") || doc_kind_is(p, "pair")) {
+ cur = p;
+ } else if (doc_kind_is(p, "assignment_expression")) {
+ TSNode pp = doc_parent(ctx, p);
+ if (doc_kind_is(pp, "expression_statement")) {
+ cur = pp;
+ }
+ }
+ }
+ if (doc_kind_is(cur, "variable_declarator")) {
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "lexical_declaration") || doc_kind_is(p, "variable_declaration")) {
+ cur = p;
+ }
+ }
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "ambient_declaration")) {
+ cur = p;
+ p = doc_parent(ctx, cur);
+ }
+ return doc_kind_is(p, "export_statement") ? p : cur;
+}
+
+/* C#: a field is documented above its field declaration, a top-level
+ * function above its global statement. */
+static TSNode doc_anchor_csharp(CBMExtractCtx *ctx, TSNode cur) {
+ if (doc_kind_is(cur, "variable_declarator")) {
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "variable_declaration")) {
+ cur = p;
+ }
+ }
+ if (doc_kind_is(cur, "variable_declaration")) {
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "field_declaration") || doc_kind_is(p, "event_field_declaration")) {
+ cur = p;
+ }
+ }
+ TSNode p = doc_parent(ctx, cur);
+ return doc_kind_is(p, "global_statement") ? p : cur;
+}
+
+/* C family: `typedef struct X {..} Y;` and `struct X {..} v;` are documented
+ * above the declaration. Only a specifier WITH a body is a definition; a
+ * bodiless one is a usage. */
+static TSNode doc_anchor_c(CBMExtractCtx *ctx, TSNode cur) {
+ if ((doc_kind_is(cur, "struct_specifier") || doc_kind_is(cur, "enum_specifier") ||
+ doc_kind_is(cur, "union_specifier") || doc_kind_is(cur, "class_specifier")) &&
+ !ts_node_is_null(ts_node_child_by_field_name(cur, TS_FIELD("body")))) {
+ TSNode p = doc_parent(ctx, cur);
+ if (doc_kind_is(p, "type_definition") || doc_kind_is(p, "declaration")) {
+ return p;
+ }
+ }
+ return cur;
+}
+
+/* Go: `var x = 1` (no parentheses) is documented above the declaration. */
+static TSNode doc_anchor_go(CBMExtractCtx *ctx, TSNode cur) {
+ if (doc_kind_is(cur, "var_spec") || doc_kind_is(cur, "const_spec")) {
+ TSNode p = doc_parent(ctx, cur);
+ if ((doc_kind_is(p, "var_declaration") || doc_kind_is(p, "const_declaration")) &&
+ ts_node_eq(ts_node_child(p, SECOND_CHILD_IDX), cur)) {
+ return p;
+ }
+ }
+ return cur;
+}
+
+/* The node whose leading comments document `node`. */
+static TSNode doc_anchor(CBMExtractCtx *ctx, TSNode node) {
+ switch (ctx->language) {
+ case CBM_LANG_JAVASCRIPT:
+ case CBM_LANG_TYPESCRIPT:
+ case CBM_LANG_TSX:
+ case CBM_LANG_ARKTS:
+ return doc_anchor_js(ctx, node);
+ case CBM_LANG_CSHARP:
+ return doc_anchor_csharp(ctx, node);
+ case CBM_LANG_C:
+ case CBM_LANG_CPP:
+ case CBM_LANG_CUDA:
+ case CBM_LANG_OBJC:
+ return doc_anchor_c(ctx, node);
+ case CBM_LANG_GO:
+ return doc_anchor_go(ctx, node);
+ default:
+ return node;
+ }
+}
+
+/* ── Perl POD: the =head / =item section that names the sub ── */
+
+enum { POD_HEAD = 1, POD_ITEM, POD_OVER, POD_BACK, POD_CUT, POD_OTHER };
+
+typedef struct {
+ uint32_t sb; /* command paragraph */
+ uint32_t eb;
+ uint32_t wb; /* end of the command word: its argument starts here */
+ uint8_t type;
+ uint8_t level; /* =headN: N; =item and =back: list depth */
+} pod_cmd_t;
+
+typedef struct {
+ const char *name;
+ size_t name_len;
+ uint32_t sb;
+ uint32_t eb;
+} pod_sec_t;
+
+typedef struct {
+ pod_sec_t *items;
+ int count;
+ int cap;
+} pod_index_t;
+
+static bool pod_line_is_blank(const char *src, uint32_t b, uint32_t e) {
+ for (uint32_t k = b; k < e; k++) {
+ if (src[k] != ' ' && src[k] != '\t' && src[k] != '\r') {
+ return false;
+ }
+ }
+ return true;
+}
+
+/* Next POD paragraph (a maximal run of non-blank lines) in src[*pos, pe). */
+static bool pod_next_para(const char *src, uint32_t *pos, uint32_t pe, uint32_t *psb,
+ uint32_t *peb) {
+ uint32_t i = *pos;
+ bool in = false;
+ uint32_t s = 0;
+ while (i < pe) {
+ uint32_t e = i;
+ while (e < pe && src[e] != '\n') {
+ e++;
+ }
+ bool blank = pod_line_is_blank(src, i, e);
+ if (!blank && !in) {
+ in = true;
+ s = i;
+ }
+ if (blank && in) {
+ *psb = s;
+ *peb = i;
+ *pos = i;
+ return true;
+ }
+ i = e < pe ? e + SKIP_ONE : pe;
+ }
+ *pos = pe;
+ if (in) {
+ *psb = s;
+ *peb = pe;
+ }
+ return in;
+}
+
+static bool pod_word_is(const char *w, size_t wl, const char *lit) {
+ return wl == strlen(lit) && strncmp(w, lit, wl) == 0;
+}
+
+/* Classify one command paragraph; `depth` is the =over nesting. */
+static pod_cmd_t pod_classify(const char *src, uint32_t sb, uint32_t eb, int *depth) {
+ uint32_t wb = sb + SKIP_ONE;
+ while (wb < eb && (isalnum((unsigned char)src[wb]) || src[wb] == '_')) {
+ wb++;
+ }
+ const char *w = src + sb + SKIP_ONE;
+ size_t wl = wb - sb - SKIP_ONE;
+ pod_cmd_t c = {sb, eb, wb, POD_OTHER, 0};
+ size_t head_len = strlen("head");
+ if (wl == head_len + SKIP_ONE && strncmp(w, "head", head_len) == 0 &&
+ isdigit((unsigned char)w[head_len])) {
+ c.type = POD_HEAD;
+ c.level = (uint8_t)(w[head_len] - '0');
+ } else if (pod_word_is(w, wl, "item")) {
+ c.type = POD_ITEM;
+ c.level = (uint8_t)*depth;
+ } else if (pod_word_is(w, wl, "over")) {
+ c.type = POD_OVER;
+ (*depth)++;
+ } else if (pod_word_is(w, wl, "back")) {
+ c.type = POD_BACK;
+ c.level = (uint8_t)*depth;
+ if (*depth > 0) {
+ (*depth)--;
+ }
+ } else if (pod_word_is(w, wl, "cut")) {
+ c.type = POD_CUT;
+ }
+ return c;
+}
+
+/* Command paragraphs of one POD block, in order. */
+static int pod_commands(CBMArena *a, const char *src, uint32_t pb, uint32_t pe, pod_cmd_t **out) {
+ int cap = DOC_SPAN_INIT_CAP;
+ int n = 0;
+ pod_cmd_t *cmds = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(pod_cmd_t));
+ *out = cmds;
+ if (!cmds) {
+ return 0;
+ }
+ int depth = 0;
+ uint32_t pos = pb;
+ uint32_t sb = 0;
+ uint32_t eb = 0;
+ while (pod_next_para(src, &pos, pe, &sb, &eb)) {
+ if (src[sb] != '=' || sb + SKIP_ONE >= eb || !isalpha((unsigned char)src[sb + SKIP_ONE])) {
+ continue;
+ }
+ if (n == cap) {
+ int ncap = cap * DOC_SPAN_GROW;
+ pod_cmd_t *grown = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_cmd_t));
+ if (!grown) {
+ break;
+ }
+ memcpy(grown, cmds, (size_t)n * sizeof(pod_cmd_t));
+ cmds = grown;
+ cap = ncap;
+ *out = cmds;
+ }
+ cmds[n++] = pod_classify(src, sb, eb, &depth);
+ }
+ return n;
+}
+
+/* End byte of the section that cmds[ci] (a =head or =item) opens. */
+static uint32_t pod_section_end(const pod_cmd_t *cmds, int n, int ci, uint32_t pe) {
+ const pod_cmd_t *c = &cmds[ci];
+ for (int j = ci + SKIP_ONE; j < n; j++) {
+ const pod_cmd_t *d = &cmds[j];
+ if (d->type == POD_CUT) {
+ return d->sb;
+ }
+ if (d->type == POD_HEAD && (c->type == POD_ITEM || d->level <= c->level)) {
+ return d->sb;
+ }
+ if (c->type == POD_ITEM && (d->type == POD_ITEM || d->type == POD_BACK) &&
+ d->level == c->level) {
+ return d->sb;
+ }
+ }
+ return pe;
+}
+
+/* A heading's text with POD formatting codes (X<..>, X<< .. >>) removed and
+ * newlines and tabs as spaces. */
+static char *pod_plain_heading(CBMArena *a, const char *src, uint32_t b, uint32_t e) {
+ char *buf = (char *)cbm_arena_alloc(a, (size_t)(e - b) + SKIP_ONE);
+ if (!buf) {
+ return NULL;
+ }
+ size_t w = 0;
+ for (uint32_t i = b; i < e; i++) {
+ char c = src[i];
+ if (c >= 'A' && c <= 'Z' && i + SKIP_ONE < e && src[i + SKIP_ONE] == '<') {
+ uint32_t j = i + SKIP_ONE;
+ while (j < e && src[j] == '<') {
+ j++;
+ }
+ if (j - i > PAIR_LEN && j < e && (src[j] == ' ' || src[j] == '\t')) {
+ j++; /* X<< text >> */
+ }
+ i = j - SKIP_ONE;
+ continue;
+ }
+ if (c == '>' && !(w > 0 && buf[w - SKIP_ONE] == '-')) {
+ continue;
+ }
+ buf[w++] = (c == '\n' || c == '\r' || c == '\t') ? ' ' : c;
+ }
+ buf[w] = '\0';
+ return buf;
+}
+
+static const char *pod_skip_spaces(const char *p) {
+ while (*p == ' ') {
+ p++;
+ }
+ return p;
+}
+
+/* The sub a heading names: after a list bullet ("*", "1."), a `$obj->`
+ * receiver and a sigil, the identifier that follows. Returns its length
+ * (0: none). */
+static size_t pod_heading_name(CBMArena *a, const char *src, uint32_t b, uint32_t e,
+ const char **name) {
+ const char *buf = pod_plain_heading(a, src, b, e);
+ if (!buf) {
+ return 0;
+ }
+ const char *p = pod_skip_spaces(buf);
+ if (*p == '*') {
+ p++;
+ } else if (isdigit((unsigned char)*p)) {
+ while (isdigit((unsigned char)*p)) {
+ p++;
+ }
+ if (*p == '.') {
+ p++;
+ }
+ }
+ p = pod_skip_spaces(p);
+ if (*p == '$') {
+ const char *q = p + SKIP_ONE;
+ while (isalnum((unsigned char)*q) || *q == '_') {
+ q++;
+ }
+ q = pod_skip_spaces(q);
+ if (q > p + SKIP_ONE && q[0] == '-' && q[SKIP_ONE] == '>') {
+ p = pod_skip_spaces(q + PAIR_LEN);
+ }
+ }
+ if (*p == '$' || *p == '@' || *p == '%' || *p == '&') {
+ p++;
+ }
+ if (!(isalpha((unsigned char)*p) || *p == '_')) {
+ return 0;
+ }
+ const char *s = p;
+ while (isalnum((unsigned char)*p) || *p == '_') {
+ p++;
+ }
+ *name = s;
+ return (size_t)(p - s);
+}
+
+static uint32_t pod_trim_end(const char *src, uint32_t sb, uint32_t eb) {
+ while (eb > sb && (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r' ||
+ src[eb - SKIP_ONE] == ' ' || src[eb - SKIP_ONE] == '\t')) {
+ eb--;
+ }
+ return eb;
+}
+
+static void pod_index_push(CBMArena *a, pod_index_t *ix, const pod_sec_t *sec) {
+ if (ix->count == ix->cap) {
+ int ncap = ix->cap ? ix->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP;
+ pod_sec_t *grown = (pod_sec_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_sec_t));
+ if (!grown) {
+ return;
+ }
+ if (ix->count > 0) {
+ memcpy(grown, ix->items, (size_t)ix->count * sizeof(pod_sec_t));
+ }
+ ix->items = grown;
+ ix->cap = ncap;
+ }
+ ix->items[ix->count++] = *sec;
+}
+
+/* Index the named =head / =item sections of one POD block. */
+static void pod_index_block(CBMExtractCtx *ctx, pod_index_t *ix, TSNode pod) {
+ CBMArena *a = doc_scratch(ctx);
+ uint32_t pb = ts_node_start_byte(pod);
+ uint32_t pe = ts_node_end_byte(pod);
+ pod_cmd_t *cmds = NULL;
+ int n = pod_commands(a, ctx->source, pb, pe, &cmds);
+ for (int ci = 0; ci < n; ci++) {
+ if (cmds[ci].type != POD_HEAD && cmds[ci].type != POD_ITEM) {
+ continue;
+ }
+ const char *nm = NULL;
+ size_t nl = pod_heading_name(a, ctx->source, cmds[ci].wb, cmds[ci].eb, &nm);
+ if (nl == 0) {
+ continue;
+ }
+ uint32_t se = pod_section_end(cmds, n, ci, pe);
+ pod_sec_t sec = {nm, nl, cmds[ci].sb, pod_trim_end(ctx->source, cmds[ci].sb, se)};
+ pod_index_push(a, ix, &sec);
+ }
+}
+
+/* The file's POD sections by name, built on the first Perl sub. */
+static const pod_index_t *pod_index(CBMExtractCtx *ctx) {
+ if (ctx->doc_pod_index) {
+ return (const pod_index_t *)ctx->doc_pod_index;
+ }
+ pod_index_t *ix = (pod_index_t *)cbm_arena_alloc(doc_scratch(ctx), sizeof(pod_index_t));
+ if (!ix) {
+ return NULL;
+ }
+ memset(ix, 0, sizeof(*ix));
+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);
+ if (ts_tree_cursor_goto_first_child(&cur)) {
+ do {
+ TSNode ch = ts_tree_cursor_current_node(&cur);
+ if (strcmp(ts_node_type(ch), "pod") == 0) {
+ pod_index_block(ctx, ix, ch);
+ }
+ } while (ts_tree_cursor_goto_next_sibling(&cur));
+ }
+ ts_tree_cursor_delete(&cur);
+ ctx->doc_pod_index = ix;
+ return ix;
+}
+
+static bool pod_sec_names(const pod_sec_t *s, const char *name, size_t nl) {
+ return s->name_len == nl && memcmp(s->name, name, nl) == 0;
+}
+
+/* Every POD section whose =head / =item names the sub exactly, joined in
+ * document order; NULL when none does. */
+static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) {
+ const pod_index_t *ix = pod_index(ctx);
+ if (!ix || !name || !name[0]) {
+ return NULL;
+ }
+ size_t nl = strlen(name);
+ size_t total = 0;
+ for (int i = 0; i < ix->count; i++) {
+ if (pod_sec_names(&ix->items[i], name, nl)) {
+ total += (size_t)(ix->items[i].eb - ix->items[i].sb) + PAIR_LEN;
+ }
+ }
+ if (total == 0) {
+ return NULL;
+ }
+ char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE);
+ if (!buf) {
+ return NULL;
+ }
+ size_t w = 0;
+ for (int i = 0; i < ix->count; i++) {
+ const pod_sec_t *s = &ix->items[i];
+ if (!pod_sec_names(s, name, nl)) {
+ continue;
+ }
+ if (w > 0) {
+ buf[w++] = '\n';
+ buf[w++] = '\n';
+ }
+ memcpy(buf + w, ctx->source + s->sb, s->eb - s->sb);
+ w += s->eb - s->sb;
+ }
+ buf[w] = '\0';
+ return buf;
+}
+
+/* ── Doc entry points ── */
+
+/* go/doc: a type spec's own doc, else its declaration's (the doc of a
+ * single-spec `type X ...` sits above the `type` keyword). */
+static const char *doc_go_type(CBMExtractCtx *ctx, TSNode node) {
+ const char *own = doc_for_anchor(ctx, node);
+ TSNode parent = doc_parent(ctx, node);
+ if (own || !doc_kind_is(parent, "type_declaration")) {
+ return own;
+ }
+ return doc_for_anchor(ctx, parent);
+}
+
+/* Doc of a Function/Method/Class-like definition. */
+static const char *extract_docstring(CBMExtractCtx *ctx, TSNode node, const char *name) {
+ CBMLanguage lang = ctx->language;
+ if (lang == CBM_LANG_PERL) {
+ const char *pod = pod_doc_by_name(ctx, name);
+ if (pod) {
+ return pod;
+ }
+ }
+ if (lang == CBM_LANG_GO &&
+ (doc_kind_is(node, "type_spec") || doc_kind_is(node, "type_alias"))) {
+ return doc_go_type(ctx, node);
+ }
+ const char *doc = doc_for_anchor(ctx, doc_anchor(ctx, node));
+ if (!doc && lang == CBM_LANG_PYTHON) {
+ return extract_python_docstring(ctx->arena, node, ctx->source);
+ }
+ return doc;
+}
+
+/* Doc of a Field, Variable, enum member or Macro (code languages only). */
+static const char *extract_member_docstring(CBMExtractCtx *ctx, TSNode node) {
+ if (!doc_lang_member_docs(ctx->language)) {
+ return NULL;
+ }
+ return doc_for_anchor(ctx, doc_anchor(ctx, node));
+}
+
+/* Go package comment: the comment group touching `package`, directives
+ * dropped (go/doc). */
+static const char *doc_go_package(CBMExtractCtx *ctx) {
+ doc_trivia_t t;
+ memset(&t, 0, sizeof(t));
+ TSNode pkg = {0};
+ bool found = false;
+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);
+ if (ts_tree_cursor_goto_first_child(&cur)) {
+ do {
+ TSNode ch = ts_tree_cursor_current_node(&cur);
+ const char *k = ts_node_type(ch);
+ if (strcmp(k, "package_clause") == 0) {
+ pkg = ch;
+ found = true;
+ break;
+ }
+ if (is_comment_node(k)) {
+ doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT);
+ doc_push_span(doc_scratch(ctx), &t, &sp);
+ } else {
+ t.count = 0;
+ }
+ } while (ts_tree_cursor_goto_next_sibling(&cur));
+ }
+ ts_tree_cursor_delete(&cur);
+ int last = t.count - SKIP_ONE;
+ if (!found || last < 0 || ts_node_start_point(pkg).row > t.items[last].erow + SKIP_ONE) {
+ return NULL;
+ }
+ int first = last;
+ while (first > 0 && t.items[first].srow <= t.items[first - SKIP_ONE].erow + SKIP_ONE) {
+ first--;
+ }
+ return doc_run_text(ctx, &t, first, last, true);
+}
+
+/* Rust inner docs (//! and the bang block) before the first item. */
+static const char *doc_rust_inner(CBMExtractCtx *ctx) {
+ doc_trivia_t t;
+ memset(&t, 0, sizeof(t));
+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);
+ if (ts_tree_cursor_goto_first_child(&cur)) {
+ do {
+ TSNode ch = ts_tree_cursor_current_node(&cur);
+ const char *k = ts_node_type(ch);
+ if (is_comment_node(k)) {
+ doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT);
+ if (sp.style == DOC_CS_LINE_BANG || sp.style == DOC_CS_BLOCK_BANG) {
+ doc_push_span(doc_scratch(ctx), &t, &sp);
+ }
+ continue;
+ }
+ if (strcmp(k, "inner_attribute_item") != 0) {
+ break; /* the first item: the inner docs are over */
+ }
+ } while (ts_tree_cursor_goto_next_sibling(&cur));
+ }
+ ts_tree_cursor_delete(&cur);
+ return t.count > 0 ? doc_run_text(ctx, &t, 0, t.count - SKIP_ONE, false) : NULL;
+}
+
+/* A file's own doc, for its File node: the Go package comment or the Rust
+ * inner docs. */
+static const char *extract_module_doc(CBMExtractCtx *ctx) {
+ if (ctx->language == CBM_LANG_GO) {
+ return doc_go_package(ctx);
+ }
+ return ctx->language == CBM_LANG_RUST ? doc_rust_inner(ctx) : NULL;
}
static int find_jvm_modifiers(TSNode node, CBMLanguage lang, TSNode *out, int max);
@@ -4124,7 +5487,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
def.is_test = def.is_test || ctx->result->is_test_file;
// Docstring
- def.docstring = extract_docstring(a, node, ctx->source, ctx->language);
+ def.docstring = extract_docstring(ctx, node, name);
// Complexity
if (spec->branching_node_types && spec->branching_node_types[0]) {
@@ -4248,8 +5611,8 @@ static void utf8_trim_partial_tail(char *text) {
}
/* Collapse `len` bytes of raw prose into a single-spaced value capped at
- * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which
- * leaves room inside the 2 KB properties buffer that carries it).
+ * MAX_COMMENT_LEN (500 bytes, which leaves room inside the 2 KB properties
+ * buffer that carries it).
*
* The output buffer is FIXED at that cap, so a section body of any size costs a
* bounded copy rather than a copy of the whole section. Returns NULL when
@@ -4953,7 +6316,7 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
def.is_exported = cbm_is_exported(name, ctx->language);
def.base_classes = extract_base_classes(a, node, ctx->source, ctx->language);
def.decorators = extract_decorators(a, node, ctx->source, ctx->language, spec);
- def.docstring = extract_docstring(a, node, ctx->source, ctx->language);
+ def.docstring = extract_docstring(ctx, node, name);
cbm_defs_push(&ctx->result->defs, a, def);
@@ -5324,7 +6687,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node,
def.route_path = prefix ? join_route_paths(a, prefix, def.route_path) : NULL;
def.route_method = prefix ? def.route_method : NULL;
}
- def.docstring = extract_docstring(a, child, ctx->source, ctx->language);
+ def.docstring = extract_docstring(ctx, child, name);
if (spec->branching_node_types && spec->branching_node_types[0]) {
set_def_complexity(&def, child, spec);
@@ -5565,6 +6928,7 @@ static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
// MinHash fingerprint
compute_fingerprint(ctx, &def, child);
+ def.docstring = extract_docstring(ctx, child, name);
cbm_defs_push(&ctx->result->defs, a, def);
}
@@ -5718,6 +7082,7 @@ static void push_var_def_qn(CBMExtractCtx *ctx, const char *name, const char *qn
def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;
def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;
def.is_exported = cbm_is_exported(name, ctx->language);
+ def.docstring = extract_member_docstring(ctx, node);
cbm_defs_push(&ctx->result->defs, a, def);
}
@@ -5851,6 +7216,7 @@ static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *cl
mdef.file_path = ctx->rel_path;
mdef.start_line = ts_node_start_point(member).row + TS_LINE_OFFSET;
mdef.end_line = ts_node_end_point(member).row + TS_LINE_OFFSET;
+ mdef.docstring = extract_member_docstring(ctx, member);
cbm_defs_push(&ctx->result->defs, a, mdef);
}
}
@@ -7447,6 +8813,7 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch
* TS @Input()-style fields, JVM field annotations via the modifiers
* child) — same extraction the class and method paths already run. */
def.decorators = extract_decorators(a, child, ctx->source, ctx->language, spec);
+ def.docstring = extract_member_docstring(ctx, child);
cbm_defs_push(&ctx->result->defs, a, def);
}
@@ -7998,6 +9365,7 @@ static void extract_c_macro_def(CBMExtractCtx *ctx, TSNode node) {
if (!ts_node_is_null(params)) {
def.signature = cbm_node_text(a, params, ctx->source);
}
+ def.docstring = extract_member_docstring(ctx, node);
cbm_defs_push(&ctx->result->defs, a, def);
}
@@ -8551,6 +9919,7 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
}
int mod_idx = ctx->result->defs.count;
cbm_defs_push(&ctx->result->defs, a, mod);
+ ctx->result->module_doc = extract_module_doc(ctx);
cbm_extract_definitions_without_module(ctx);
diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c
index 90c8c6d82..bd8b9b8a9 100644
--- a/internal/cbm/result_compact.c
+++ b/internal/cbm/result_compact.c
@@ -385,6 +385,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {
cr_list(c, &r->macros);
cr_str(c, &r->error_msg);
cr_str(c, &r->error_ranges);
+ cr_str(c, &r->module_doc);
cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0);
}
diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c
index 804ffba65..f3a2cd948 100644
--- a/src/pipeline/pass_definitions.c
+++ b/src/pipeline/pass_definitions.c
@@ -22,6 +22,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
#include "pipeline/pipeline_internal.h"
#include "graph_buffer/graph_buffer.h"
#include "foundation/log.h"
+#include "foundation/mem_core.h" /* cbm_alloc/cbm_free: the grown properties buffer */
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/limits.h"
@@ -321,16 +322,74 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
}
}
-/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges. */
-static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel) {
+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
+ * serialized docstring field, which has no length cap (a field that does not
+ * fit is dropped whole). Returns `stack` for a def without a docstring, or
+ * when the larger buffer cannot be allocated. Twin of pass_parallel.c -- keep
+ * both in sync. */
+static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
+ if (!def->docstring || !def->docstring[0]) {
+ return stack;
+ }
+ size_t need =
+ *size + strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD;
+ char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);
+ if (!buf) {
+ return stack;
+ }
+ *size = need;
+ return buf;
+}
+
+/* Add a file's own doc (Go package comment, Rust inner docs) to its File
+ * node as "docstring". Twin of pass_parallel.c -- keep both in sync. */
+static void pd_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {
+ if (!file_node || !doc || !doc[0]) {
+ return;
+ }
+ const char *old = file_node->properties_json ? file_node->properties_json : "{}";
+ size_t olen = strlen(old);
+ if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {
+ return; /* not a JSON object -- leave it untouched */
+ }
+ size_t cap = olen + strlen("docstring") + def_json_escaped_len(doc) + PD_JSON_FIELD_OVERHEAD +
+ PD_ESC_SPACE + SKIP_ONE;
+ char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);
+ if (!neu) {
+ return;
+ }
+ size_t pos = olen - SKIP_ONE; /* without the closing brace */
+ memcpy(neu, old, pos);
+ neu[pos] = '\0';
+ append_json_string(neu, cap, &pos, "docstring", doc);
+ if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */
+ memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);
+ pos--;
+ }
+ neu[pos++] = '}';
+ neu[pos] = '\0';
+ (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);
+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);
+}
+
+/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges.
+ * `file_doc`, the file's own doc (passed with its first def only), goes on the
+ * File node. */
+static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel,
+ const char *file_doc) {
if (!def->qualified_name || !def->name) {
return;
}
- char props[CBM_SZ_2K];
- build_def_props(props, sizeof(props), def);
+ char stack[CBM_SZ_2K];
+ size_t props_size = sizeof(stack);
+ char *props = pd_props_buf(def, stack, &props_size);
+ build_def_props(props, props_size, def);
int64_t node_id = cbm_gbuf_upsert_node(
ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name,
def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props);
+ if (props != stack) {
+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
+ }
/* Registry membership is defined ONCE by cbm_label_is_registry_symbol
* (helpers.c): callables + type-like containers (INHERITS/IMPLEMENTS/method/
* field resolution), Variable/Field (READS/WRITES resolution), and Table/View
@@ -341,6 +400,7 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const
}
char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__");
const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);
+ pd_add_file_doc(file_node, file_doc);
if (file_node && node_id > 0) {
cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}");
}
@@ -860,8 +920,10 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
}
/* Create nodes for each definition */
+ const char *file_doc = result->module_doc; /* goes with the first def */
for (int d = 0; d < result->defs.count; d++) {
- process_def(ctx, &result->defs.items[d], rel);
+ process_def(ctx, &result->defs.items[d], rel, file_doc);
+ file_doc = NULL;
total_defs++;
}
diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c
index 9df46c1af..0c12e6c4a 100644
--- a/src/pipeline/pass_parallel.c
+++ b/src/pipeline/pass_parallel.c
@@ -697,15 +697,39 @@ typedef struct {
* to the response/logfile — this only throttles the stderr noise). */
enum { PP_OVERSIZED_WARN_MAX = 32 };
+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
+ * serialized docstring field, which has no length cap (a field that does not
+ * fit is dropped whole). Returns `stack` for a def without a docstring, or
+ * when the larger buffer cannot be allocated. Twin of pass_definitions.c --
+ * keep both in sync. */
+static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
+ if (!def->docstring || !def->docstring[0]) {
+ return stack;
+ }
+ size_t need =
+ *size + strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD;
+ char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);
+ if (!buf) {
+ return stack;
+ }
+ *size = need;
+ return buf;
+}
+
/* Insert one definition node (and its route if present) into the local gbuf. */
static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_file_info_t *fi,
CBMDefinition *def) {
- char props[CBM_SZ_2K];
- build_def_props(props, sizeof(props), def);
+ char stack[CBM_SZ_2K];
+ size_t props_size = sizeof(stack);
+ char *props = pp_props_buf(def, stack, &props_size);
+ build_def_props(props, props_size, def);
int64_t func_id =
cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name,
def->qualified_name, def->file_path ? def->file_path : fi->rel_path,
(int)def->start_line, (int)def->end_line, props);
+ if (props != stack) {
+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
+ }
ws->nodes_created++;
if (def->route_path && def->route_path[0] != '\0') {
const char *rm = def->route_method ? def->route_method : "ANY";
@@ -1692,6 +1716,37 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d
return edges;
}
+/* Add a file's own doc (Go package comment, Rust inner docs) to its File
+ * node as "docstring". Twin of pass_definitions.c -- keep both in sync. */
+static void pp_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {
+ if (!file_node || !doc || !doc[0]) {
+ return;
+ }
+ const char *old = file_node->properties_json ? file_node->properties_json : "{}";
+ size_t olen = strlen(old);
+ if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {
+ return; /* not a JSON object -- leave it untouched */
+ }
+ size_t cap = olen + strlen("docstring") + pp_json_escaped_len(doc) + PP_JSON_FIELD_OVERHEAD +
+ PP_ESC_SPACE + SKIP_ONE;
+ char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);
+ if (!neu) {
+ return;
+ }
+ size_t pos = olen - SKIP_ONE; /* without the closing brace */
+ memcpy(neu, old, pos);
+ neu[pos] = '\0';
+ append_json_string(neu, cap, &pos, "docstring", doc);
+ if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */
+ memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);
+ pos--;
+ }
+ neu[pos++] = '}';
+ neu[pos] = '\0';
+ (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);
+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);
+}
+
/* Create IMPORTS edges for one file's imports (parallel path). */
static int create_imports_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result,
const char *rel, CBMHashTable *namespace_map) {
@@ -1826,6 +1881,7 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);
int64_t file_node_id = file_node ? file_node->id : 0;
free(file_qn);
+ pp_add_file_doc(file_node, result->module_doc);
for (int d = 0; d < result->defs.count; d++) {
defines_edges +=
register_and_link_def(ctx, &result->defs.items[d], file_node_id, ®_entries);
diff --git a/tests/test_extraction.c b/tests/test_extraction.c
index 0b6800cd1..fe46d7246 100644
--- a/tests/test_extraction.c
+++ b/tests/test_extraction.c
@@ -7064,12 +7064,12 @@ TEST(extract_python_method_test_dir_marks_is_test_issue1294) {
PASS();
}
-/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a
+/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) could split a
* multi-byte UTF-8 character, leaving an incomplete byte sequence.
* Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character
- * (U+6210 = 成 = e6 88 90). The raw byte truncation at offset 500 lands
- * one byte past the character start, splitting it. After the fix the
- * truncated string must end on a complete codepoint boundary. */
+ * (U+6210 = 成 = e6 88 90), where a raw byte cut at offset 500 lands one
+ * byte past the character start. Docstrings are no longer cut at all, so
+ * the whole comment is kept and still ends on a complete codepoint. */
TEST(docstring_utf8_truncation_boundary_issue1017) {
/* Build a comment: "// " (3 bytes) + 495 ASCII 'A' + "成成成" (9 bytes)
* Total comment text = 3 + 495 + 9 = 507 bytes.
@@ -7110,10 +7110,10 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {
}
ASSERT_NOT_NULL(doc);
- /* Verify every byte in the truncated docstring is valid UTF-8:
- * no trailing incomplete multi-byte sequence. */
+ /* The whole comment, and every byte of it valid UTF-8: no trailing
+ * incomplete multi-byte sequence. */
+ ASSERT_STR_EQ(doc, comment);
size_t len = strlen(doc);
- ASSERT_TRUE(len <= 500);
const unsigned char *u = (const unsigned char *)doc;
size_t i = 0;
while (i < len) {
@@ -7139,6 +7139,298 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {
PASS();
}
+/* ═══════════════════════════════════════════════════════════════════
+ * Doc comments: the whole comment run above a definition, above its
+ * attributes / export / declaration wrapper, never a detached or
+ * banner-only run, and never cut at 500 bytes.
+ * ═══════════════════════════════════════════════════════════════════ */
+
+/* The def with this label and name, or NULL. */
+static const CBMDefinition *doc_def(CBMFileResult *r, const char *label, const char *name) {
+ for (int i = 0; i < r->defs.count; i++) {
+ const CBMDefinition *d = &r->defs.items[i];
+ if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0) {
+ return d;
+ }
+ }
+ return NULL;
+}
+
+TEST(doc_go_comment_group_without_directives) {
+ CBMFileResult *r = extract("package knob\n\n"
+ "// Turn rotates the knob by one step.\n"
+ "// It is safe for concurrent use.\n"
+ "//go:noinline\n"
+ "func Turn() {}\n",
+ CBM_LANG_GO, "t", "knob.go");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Function", "Turn");
+ ASSERT_NOT_NULL(d);
+ /* go/ast CommentGroup.Text: the whole group, the //go: directive dropped. */
+ ASSERT_STR_EQ(d->docstring, "// Turn rotates the knob by one step.\n"
+ "// It is safe for concurrent use.");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_go_struct_field) {
+ CBMFileResult *r = extract("package knob\n\n"
+ "// Knob is a rotary control.\n"
+ "type Knob struct {\n"
+ "\t// Steps counts the turns so far.\n"
+ "\tSteps int\n"
+ "}\n",
+ CBM_LANG_GO, "t", "knob.go");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *knob = doc_def(r, "Struct", "Knob");
+ ASSERT_NOT_NULL(knob);
+ ASSERT_STR_EQ(knob->docstring, "// Knob is a rotary control.");
+ const CBMDefinition *steps = doc_def(r, "Field", "Steps");
+ ASSERT_NOT_NULL(steps);
+ ASSERT_STR_EQ(steps->docstring, "// Steps counts the turns so far.");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_go_detached_comment_is_not_doc) {
+ /* go/doc: a comment group separated by a blank line documents nothing. */
+ CBMFileResult *r = extract("package knob\n\n"
+ "// Section: helpers.\n"
+ "\n"
+ "func Reset() {}\n",
+ CBM_LANG_GO, "t", "knob.go");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Function", "Reset");
+ ASSERT_NOT_NULL(d);
+ ASSERT_NULL(d->docstring);
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_python_comment_run_and_detached) {
+ CBMFileResult *r = extract("# Turns the knob.\n"
+ "# One step per call.\n"
+ "def turn():\n"
+ " pass\n"
+ "\n"
+ "# Section: helpers.\n"
+ "\n"
+ "def reset():\n"
+ " pass\n",
+ CBM_LANG_PYTHON, "t", "knob.py");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *turn = doc_def(r, "Function", "turn");
+ ASSERT_NOT_NULL(turn);
+ ASSERT_STR_EQ(turn->docstring, "# Turns the knob.\n# One step per call.");
+ const CBMDefinition *reset = doc_def(r, "Function", "reset");
+ ASSERT_NOT_NULL(reset);
+ ASSERT_NULL(reset->docstring);
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_rust_attribute_impl_method_and_inner_doc) {
+ CBMFileResult *r = extract("//! The knob crate.\n"
+ "\n"
+ "/// Adds two steps.\n"
+ "/// Wraps on overflow.\n"
+ "#[inline]\n"
+ "pub fn add(a: u8, b: u8) -> u8 { a.wrapping_add(b) }\n"
+ "\n"
+ "pub struct Knob;\n"
+ "\n"
+ "impl Knob {\n"
+ " /// Turns the knob once.\n"
+ " pub fn turn(&self) {}\n"
+ "}\n",
+ CBM_LANG_RUST, "t", "lib.rs");
+ ASSERT_NOT_NULL(r);
+ /* The doc sits above the attribute; a run is joined line by line. */
+ const CBMDefinition *add = doc_def(r, "Function", "add");
+ ASSERT_NOT_NULL(add);
+ ASSERT_STR_EQ(add->docstring, "/// Adds two steps.\n/// Wraps on overflow.");
+ /* impl methods get their doc too; a single comment keeps its bytes. */
+ const CBMDefinition *turn = doc_def(r, "Method", "turn");
+ ASSERT_NOT_NULL(turn);
+ ASSERT_STR_EQ(turn->docstring, "/// Turns the knob once.\n");
+ /* The //! inner doc documents the crate (the File node), not the struct. */
+ const CBMDefinition *knob = doc_def(r, "Struct", "Knob");
+ ASSERT_NOT_NULL(knob);
+ ASSERT_NULL(knob->docstring);
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_ts_export_and_arrow_wrapper) {
+ CBMFileResult *r = extract("/** Exported helper. */\n"
+ "export function helper(): number {\n"
+ " return 1;\n"
+ "}\n"
+ "\n"
+ "/** Arrow helper. */\n"
+ "export const arrow = (): number => 2;\n",
+ CBM_LANG_TYPESCRIPT, "t", "knob.ts");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *helper = doc_def(r, "Function", "helper");
+ ASSERT_NOT_NULL(helper);
+ ASSERT_STR_EQ(helper->docstring, "/** Exported helper. */");
+ const CBMDefinition *arrow = doc_def(r, "Function", "arrow");
+ ASSERT_NOT_NULL(arrow);
+ ASSERT_STR_EQ(arrow->docstring, "/** Arrow helper. */");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_csharp_xml_doc_run_and_top_level_function) {
+ CBMFileResult *r = extract("/// Adds one.\n"
+ "static int AddOne(int x) => x + 1;\n"
+ "\n"
+ "namespace Knobs {\n"
+ " public class Knob {\n"
+ " /// \n"
+ " /// Turns the knob.\n"
+ " /// \n"
+ " public void Turn() {}\n"
+ " }\n"
+ "}\n",
+ CBM_LANG_CSHARP, "t", "Knob.cs");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *turn = doc_def(r, "Method", "Turn");
+ ASSERT_NOT_NULL(turn);
+ ASSERT_STR_EQ(turn->docstring, "/// \n/// Turns the knob.\n/// ");
+ const CBMDefinition *add = doc_def(r, "Function", "AddOne");
+ ASSERT_NOT_NULL(add);
+ ASSERT_STR_EQ(add->docstring, "/// Adds one.");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_java_javadoc_over_line_comment_and_field) {
+ CBMFileResult *r = extract("class Knob {\n"
+ " /** Turns taken so far. */\n"
+ " private int steps;\n"
+ "\n"
+ " /** Turns the knob. */\n"
+ " // NOSONAR\n"
+ " void turn() {}\n"
+ "}\n",
+ CBM_LANG_JAVA, "t", "Knob.java");
+ ASSERT_NOT_NULL(r);
+ /* A plain comment between the Javadoc and the method does not replace it. */
+ const CBMDefinition *turn = doc_def(r, "Method", "turn");
+ ASSERT_NOT_NULL(turn);
+ ASSERT_STR_EQ(turn->docstring, "/** Turns the knob. */");
+ const CBMDefinition *steps = doc_def(r, "Field", "steps");
+ ASSERT_NOT_NULL(steps);
+ ASSERT_STR_EQ(steps->docstring, "/** Turns taken so far. */");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_kotlin_kdoc_folded_after_imports) {
+ /* tree-sitter-kotlin folds this KDoc into the node before the class, so it
+ * is no sibling of it. */
+ CBMFileResult *r = extract("package knobs\n"
+ "\n"
+ "import knobs.core.Step\n"
+ "import knobs.core.Turn\n"
+ "\n"
+ "/** Base class of every knob. */\n"
+ "abstract class BaseKnob(step: Step) : Turn(step)\n",
+ CBM_LANG_KOTLIN, "t", "BaseKnob.kt");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Class", "BaseKnob");
+ ASSERT_NOT_NULL(d);
+ ASSERT_STR_EQ(d->docstring, "/** Base class of every knob. */");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_c_typedef_struct) {
+ CBMFileResult *r = extract("/* A file event. */\n"
+ "typedef struct aeFileEvent {\n"
+ " int mask;\n"
+ "} aeFileEvent;\n",
+ CBM_LANG_C, "t", "ae.h");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Class", "aeFileEvent");
+ ASSERT_NOT_NULL(d);
+ ASSERT_STR_EQ(d->docstring, "/* A file event. */");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_perl_pod_section_by_sub_name) {
+ CBMFileResult *r = extract("package Knob;\n"
+ "\n"
+ "sub turn { return 1; }\n"
+ "\n"
+ "1;\n"
+ "__END__\n"
+ "\n"
+ "=head1 METHODS\n"
+ "\n"
+ "=head2 turn\n"
+ "\n"
+ "Turns the knob once.\n"
+ "\n"
+ "=cut\n",
+ CBM_LANG_PERL, "t", "Knob.pm");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Function", "turn");
+ ASSERT_NOT_NULL(d);
+ ASSERT_STR_EQ(d->docstring, "=head2 turn\n\nTurns the knob once.");
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_long_comment_kept_whole) {
+ /* One 12-line block comment well past the old 500-byte cut, kept byte for
+ * byte. */
+ char src[4096];
+ char want[2048];
+ int w = snprintf(want, sizeof(want), "/*");
+ for (int i = 0; i < 12; i++) {
+ w += snprintf(want + w, sizeof(want) - (size_t)w,
+ "\n * Line %02d of a long doc that keeps going past the cut.", i);
+ }
+ snprintf(want + w, sizeof(want) - (size_t)w, "\n */");
+ snprintf(src, sizeof(src), "package knob\n\n%s\nfunc Long() {}\n", want);
+ ASSERT_GT((int)strlen(want), 500);
+
+ CBMFileResult *r = extract(src, CBM_LANG_GO, "t", "knob.go");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *d = doc_def(r, "Function", "Long");
+ ASSERT_NOT_NULL(d);
+ ASSERT_STR_EQ(d->docstring, want);
+ cbm_free_result(r);
+ PASS();
+}
+
+TEST(doc_divider_only_run_is_not_doc) {
+ CBMFileResult *r = extract("/*****************************/\n"
+ "void knob(void) {}\n"
+ "\n"
+ "// \xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\n"
+ "void dial(void) {}\n"
+ "\n"
+ "/* Resets the dial. */\n"
+ "void reset(void) {}\n",
+ CBM_LANG_C, "t", "knob.c");
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *knob = doc_def(r, "Function", "knob");
+ ASSERT_NOT_NULL(knob);
+ ASSERT_NULL(knob->docstring);
+ const CBMDefinition *dial = doc_def(r, "Function", "dial"); /* a box-drawing banner */
+ ASSERT_NOT_NULL(dial);
+ ASSERT_NULL(dial->docstring);
+ const CBMDefinition *reset = doc_def(r, "Function", "reset");
+ ASSERT_NOT_NULL(reset);
+ ASSERT_STR_EQ(reset->docstring, "/* Resets the dial. */");
+ cbm_free_result(r);
+ PASS();
+}
+
/* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file
* whose root node has hundreds of thousands of FLAT SIBLINGS (580k ////
* comment lines in the 3.5 MB fourslash fixture) hung extraction for over
@@ -9291,6 +9583,19 @@ SUITE(extraction) {
RUN_TEST(extract_c_test_dir_marks_is_test_issue1294);
RUN_TEST(extract_python_method_test_dir_marks_is_test_issue1294);
RUN_TEST(docstring_utf8_truncation_boundary_issue1017);
+ RUN_TEST(doc_go_comment_group_without_directives);
+ RUN_TEST(doc_go_struct_field);
+ RUN_TEST(doc_go_detached_comment_is_not_doc);
+ RUN_TEST(doc_python_comment_run_and_detached);
+ RUN_TEST(doc_rust_attribute_impl_method_and_inner_doc);
+ RUN_TEST(doc_ts_export_and_arrow_wrapper);
+ RUN_TEST(doc_csharp_xml_doc_run_and_top_level_function);
+ RUN_TEST(doc_java_javadoc_over_line_comment_and_field);
+ RUN_TEST(doc_kotlin_kdoc_folded_after_imports);
+ RUN_TEST(doc_c_typedef_struct);
+ RUN_TEST(doc_perl_pod_section_by_sub_name);
+ RUN_TEST(doc_long_comment_kept_whole);
+ RUN_TEST(doc_divider_only_run_is_not_doc);
RUN_TEST(extract_ts_decorators_survive_interleaved_comment);
/* #518/#519 — prose carried into docstring so nodes_fts can index it */
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 56be6b88e..9a4c461c6 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -9578,6 +9578,83 @@ TEST(pipeline_docstring_go_class) {
PASS();
}
+/* Index one file; true when the node with this label and name stores `want` in
+ * its properties JSON. */
+static bool doc_props_contain(const char *file, const char *content, const char *label,
+ const char *name, const char *want) {
+ const char *files[] = {file};
+ const char *contents[] = {content};
+ if (setup_lang_repo(files, contents, 1) != 0) {
+ return false;
+ }
+ char db[512];
+ snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir);
+ cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL);
+ bool found = false;
+ if (p && cbm_pipeline_run(p) == 0) {
+ cbm_store_t *s = cbm_store_open_path(db);
+ cbm_node_t *nodes = NULL;
+ int nc = 0;
+ if (s && cbm_store_find_nodes_by_label(s, cbm_pipeline_project_name(p), label, &nodes,
+ &nc) == 0) {
+ for (int i = 0; i < nc; i++) {
+ if (nodes[i].name && strcmp(nodes[i].name, name) == 0 && nodes[i].properties_json &&
+ strstr(nodes[i].properties_json, want)) {
+ found = true;
+ }
+ }
+ cbm_store_free_nodes(nodes, nc);
+ }
+ if (s) {
+ cbm_store_close(s);
+ }
+ }
+ if (p) {
+ cbm_pipeline_free(p);
+ }
+ teardown_lang_repo();
+ return found;
+}
+
+TEST(pipeline_doc_go_package_comment_on_file_node) {
+ /* go/doc: the comment group touching `package` documents the package; it is
+ * stored on the File node. */
+ ASSERT_TRUE(doc_props_contain("knob.go",
+ "// Package knob turns knobs.\n"
+ "package knob\n\n"
+ "func Turn() {}\n",
+ "File", "knob.go",
+ "\"docstring\":\"// Package knob turns knobs.\""));
+ PASS();
+}
+
+TEST(pipeline_doc_rust_inner_doc_on_file_node) {
+ ASSERT_TRUE(doc_props_contain("lib.rs",
+ "//! Knob crate.\n"
+ "//! Turns knobs.\n"
+ "\n"
+ "pub fn turn() {}\n",
+ "File", "lib.rs",
+ "\"docstring\":\"//! Knob crate.\\n//! Turns knobs.\""));
+ PASS();
+}
+
+TEST(pipeline_doc_long_docstring_stored_whole) {
+ /* A 30-line doc (longer than the old fixed 2 KB properties buffer on its
+ * own) is stored whole, and the fields after it are not dropped. */
+ char src[4096];
+ int s = snprintf(src, sizeof(src), "package knob\n\n");
+ for (int i = 0; i < 30; i++) {
+ s += snprintf(src + s, sizeof(src) - (size_t)s,
+ "// Line %02d of a long doc comment that keeps on going.\n", i);
+ }
+ snprintf(src + s, sizeof(src) - (size_t)s, "func Long(steps int) int { return steps }\n");
+ ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long",
+ "that keeps on going.\\n// Line 29 of a long doc"));
+ ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long", "\"signature\":"));
+ PASS();
+}
+
TEST(project_name_from_path) {
/* Port of TestProjectNameFromPath — more cases than integ_pipeline_project_name */
struct {
@@ -16405,6 +16482,9 @@ SUITE(pipeline) {
RUN_TEST(pipeline_docstring_java_method);
RUN_TEST(pipeline_docstring_kotlin_function);
RUN_TEST(pipeline_docstring_go_class);
+ RUN_TEST(pipeline_doc_go_package_comment_on_file_node);
+ RUN_TEST(pipeline_doc_rust_inner_doc_on_file_node);
+ RUN_TEST(pipeline_doc_long_docstring_stored_whole);
/* Project name */
RUN_TEST(project_name_from_path);
RUN_TEST(project_name_drive_letter_case_insensitive_issue394);