Merge pull request #2459 from DeusData/fix/doc-comment-capture

fix(extract): capture complete doc comments across languages

internal/cbm/cbm.h

@@ -642,6 +642,11 @@ typedef struct CBMFileResult {
     // by cbm_free_result(); ordinary single-file results leave these zeroed.
     struct CBMFileResult **owned_results;
     int owned_result_count;
+
+    /* The file's own doc, set on its File node: the Go package comment or
+     * the Rust inner docs (//!). NULL for other languages and undocumented
+     * files. */
+    const char *module_doc;
 } CBMFileResult;
 
 // --- Enclosing function cache ---
@@ -728,6 +733,11 @@ typedef struct {
     /* How many nodes the unified walk actually visited (whether or not it ran
      * out of budget) — the measurement the budget has to be expressed in. */
     uint32_t walk_nodes_visited;
+    /* Doc-comment lookup state (extract_defs.c), NULL until first used and
+     * allocated in `scratch`: the memo of parents' child arrays and the Perl
+     * POD section index. */
+    void *doc_memo;
+    void *doc_pod_index;
 } CBMExtractCtx;
 
 // --- Public API ---

internal/cbm/result_compact.c

@@ -392,6 +392,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {
     cr_list(c, &r->macros);
     cr_str(c, &r->error_msg);
     cr_str(c, &r->error_ranges);
+    cr_str(c, &r->module_doc);
     cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0);
 }
 

src/pipeline/pass_definitions.c

@@ -22,6 +22,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };
 #include "pipeline/pipeline_internal.h"
 #include "graph_buffer/graph_buffer.h"
 #include "foundation/log.h"
+#include "foundation/mem_core.h" /* cbm_alloc/cbm_free: the grown properties buffer */
 #include "foundation/compat.h"
 #include "foundation/compat_fs.h"
 #include "foundation/limits.h"
@@ -321,16 +322,74 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
     }
 }
 
-/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges. */
-static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel) {
+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
+ * serialized docstring field, which has no length cap (a field that does not
+ * fit is dropped whole). Returns `stack` for a def without a docstring, or
+ * when the larger buffer cannot be allocated. Twin of pass_parallel.c -- keep
+ * both in sync. */
+static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
+    if (!def->docstring || !def->docstring[0]) {
+        return stack;
+    }
+    size_t need =
+        *size + strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD;
+    char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);
+    if (!buf) {
+        return stack;
+    }
+    *size = need;
+    return buf;
+}
+
+/* Add a file's own doc (Go package comment, Rust inner docs) to its File
+ * node as "docstring". Twin of pass_parallel.c -- keep both in sync. */
+static void pd_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {
+    if (!file_node || !doc || !doc[0]) {
+        return;
+    }
+    const char *old = file_node->properties_json ? file_node->properties_json : "{}";
+    size_t olen = strlen(old);
+    if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {
+        return; /* not a JSON object -- leave it untouched */
+    }
+    size_t cap = olen + strlen("docstring") + def_json_escaped_len(doc) + PD_JSON_FIELD_OVERHEAD +
+                 PD_ESC_SPACE + SKIP_ONE;
+    char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);
+    if (!neu) {
+        return;
+    }
+    size_t pos = olen - SKIP_ONE; /* without the closing brace */
+    memcpy(neu, old, pos);
+    neu[pos] = '\0';
+    append_json_string(neu, cap, &pos, "docstring", doc);
+    if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */
+        memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);
+        pos--;
+    }
+    neu[pos++] = '}';
+    neu[pos] = '\0';
+    (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);
+    cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);
+}
+
+/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges.
+ * `file_doc`, the file's own doc (passed with its first def only), goes on the
+ * File node. */
+static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel,
+                        const char *file_doc) {
     if (!def->qualified_name || !def->name) {
         return;
     }
-    char props[CBM_SZ_2K];
-    build_def_props(props, sizeof(props), def);
+    char stack[CBM_SZ_2K];
+    size_t props_size = sizeof(stack);
+    char *props = pd_props_buf(def, stack, &props_size);
+    build_def_props(props, props_size, def);
     int64_t node_id = cbm_gbuf_upsert_node(
         ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name,
         def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props);
+    if (props != stack) {
+        cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
+    }
     /* Registry membership is defined ONCE by cbm_label_is_registry_symbol
      * (helpers.c): callables + type-like containers (INHERITS/IMPLEMENTS/method/
      * field resolution), Variable/Field (READS/WRITES resolution), and Table/View
@@ -341,6 +400,7 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const
     }
     char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__");
     const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);
+    pd_add_file_doc(file_node, file_doc);
     if (file_node && node_id > 0) {
         cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}");
     }
@@ -860,8 +920,10 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
         }
 
         /* Create nodes for each definition */
+        const char *file_doc = result->module_doc; /* goes with the first def */
         for (int d = 0; d < result->defs.count; d++) {
-            process_def(ctx, &result->defs.items[d], rel);
+            process_def(ctx, &result->defs.items[d], rel, file_doc);
+            file_doc = NULL;
             total_defs++;
         }
 

src/pipeline/pass_parallel.c

@@ -698,15 +698,39 @@ typedef struct {
  * to the response/logfile — this only throttles the stderr noise). */
 enum { PP_OVERSIZED_WARN_MAX = 32 };
 
+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole
+ * serialized docstring field, which has no length cap (a field that does not
+ * fit is dropped whole). Returns `stack` for a def without a docstring, or
+ * when the larger buffer cannot be allocated. Twin of pass_definitions.c --
+ * keep both in sync. */
+static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
+    if (!def->docstring || !def->docstring[0]) {
+        return stack;
+    }
+    size_t need =
+        *size + strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD;
+    char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);
+    if (!buf) {
+        return stack;
+    }
+    *size = need;
+    return buf;
+}
+
 /* Insert one definition node (and its route if present) into the local gbuf. */
 static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_file_info_t *fi,
                                  CBMDefinition *def) {
-    char props[CBM_SZ_2K];
-    build_def_props(props, sizeof(props), def);
+    char stack[CBM_SZ_2K];
+    size_t props_size = sizeof(stack);
+    char *props = pp_props_buf(def, stack, &props_size);
+    build_def_props(props, props_size, def);
     int64_t func_id =
         cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name,
                              def->qualified_name, def->file_path ? def->file_path : fi->rel_path,
                              (int)def->start_line, (int)def->end_line, props);
+    if (props != stack) {
+        cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
+    }
     ws->nodes_created++;
     if (def->route_path && def->route_path[0] != '\0') {
         const char *rm = def->route_method ? def->route_method : "ANY";
@@ -1693,6 +1717,37 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d
     return edges;
 }
 
+/* Add a file's own doc (Go package comment, Rust inner docs) to its File
+ * node as "docstring". Twin of pass_definitions.c -- keep both in sync. */
+static void pp_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {
+    if (!file_node || !doc || !doc[0]) {
+        return;
+    }
+    const char *old = file_node->properties_json ? file_node->properties_json : "{}";
+    size_t olen = strlen(old);
+    if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {
+        return; /* not a JSON object -- leave it untouched */
+    }
+    size_t cap = olen + strlen("docstring") + pp_json_escaped_len(doc) + PP_JSON_FIELD_OVERHEAD +
+                 PP_ESC_SPACE + SKIP_ONE;
+    char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);
+    if (!neu) {
+        return;
+    }
+    size_t pos = olen - SKIP_ONE; /* without the closing brace */
+    memcpy(neu, old, pos);
+    neu[pos] = '\0';
+    append_json_string(neu, cap, &pos, "docstring", doc);
+    if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */
+        memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);
+        pos--;
+    }
+    neu[pos++] = '}';
+    neu[pos] = '\0';
+    (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);
+    cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);
+}
+
 /* Create IMPORTS edges for one file's imports (parallel path). */
 static int create_imports_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result,
                                 const char *rel, CBMHashTable *namespace_map) {
@@ -1827,6 +1882,7 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
             const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);
             int64_t file_node_id = file_node ? file_node->id : 0;
             free(file_qn);
+            pp_add_file_doc(file_node, result->module_doc);
             for (int d = 0; d < result->defs.count; d++) {
                 defines_edges +=
                     register_and_link_def(ctx, &result->defs.items[d], file_node_id, &reg_entries);

tests/test_extraction.c

@@ -7120,12 +7120,12 @@ TEST(extract_python_method_test_dir_marks_is_test_issue1294) {
     PASS();
 }
 
-/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a
+/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) could split a
  * multi-byte UTF-8 character, leaving an incomplete byte sequence.
  * Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character
- * (U+6210 = 成 = e6 88 90).  The raw byte truncation at offset 500 lands
- * one byte past the character start, splitting it.  After the fix the
- * truncated string must end on a complete codepoint boundary. */
+ * (U+6210 = 成 = e6 88 90), where a raw byte cut at offset 500 lands one
+ * byte past the character start. Docstrings are no longer cut at all, so
+ * the whole comment is kept and still ends on a complete codepoint. */
 TEST(docstring_utf8_truncation_boundary_issue1017) {
     /* Build a comment: "// " (3 bytes) + 495 ASCII 'A' + "成成成" (9 bytes)
      * Total comment text = 3 + 495 + 9 = 507 bytes.
@@ -7166,10 +7166,10 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {
     }
     ASSERT_NOT_NULL(doc);
 
-    /* Verify every byte in the truncated docstring is valid UTF-8:
-     * no trailing incomplete multi-byte sequence. */
+    /* The whole comment, and every byte of it valid UTF-8: no trailing
+     * incomplete multi-byte sequence. */
+    ASSERT_STR_EQ(doc, comment);
     size_t len = strlen(doc);
-    ASSERT_TRUE(len <= 500);
     const unsigned char *u = (const unsigned char *)doc;
     size_t i = 0;
     while (i < len) {
@@ -7195,6 +7195,298 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {
     PASS();
 }
 
+/* ═══════════════════════════════════════════════════════════════════
+ * Doc comments: the whole comment run above a definition, above its
+ * attributes / export / declaration wrapper, never a detached or
+ * banner-only run, and never cut at 500 bytes.
+ * ═══════════════════════════════════════════════════════════════════ */
+
+/* The def with this label and name, or NULL. */
+static const CBMDefinition *doc_def(CBMFileResult *r, const char *label, const char *name) {
+    for (int i = 0; i < r->defs.count; i++) {
+        const CBMDefinition *d = &r->defs.items[i];
+        if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0) {
+            return d;
+        }
+    }
+    return NULL;
+}
+
+TEST(doc_go_comment_group_without_directives) {
+    CBMFileResult *r = extract("package knob\n\n"
+                               "// Turn rotates the knob by one step.\n"
+                               "// It is safe for concurrent use.\n"
+                               "//go:noinline\n"
+                               "func Turn() {}\n",
+                               CBM_LANG_GO, "t", "knob.go");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Function", "Turn");
+    ASSERT_NOT_NULL(d);
+    /* go/ast CommentGroup.Text: the whole group, the //go: directive dropped. */
+    ASSERT_STR_EQ(d->docstring, "// Turn rotates the knob by one step.\n"
+                                "// It is safe for concurrent use.");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_go_struct_field) {
+    CBMFileResult *r = extract("package knob\n\n"
+                               "// Knob is a rotary control.\n"
+                               "type Knob struct {\n"
+                               "\t// Steps counts the turns so far.\n"
+                               "\tSteps int\n"
+                               "}\n",
+                               CBM_LANG_GO, "t", "knob.go");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *knob = doc_def(r, "Struct", "Knob");
+    ASSERT_NOT_NULL(knob);
+    ASSERT_STR_EQ(knob->docstring, "// Knob is a rotary control.");
+    const CBMDefinition *steps = doc_def(r, "Field", "Steps");
+    ASSERT_NOT_NULL(steps);
+    ASSERT_STR_EQ(steps->docstring, "// Steps counts the turns so far.");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_go_detached_comment_is_not_doc) {
+    /* go/doc: a comment group separated by a blank line documents nothing. */
+    CBMFileResult *r = extract("package knob\n\n"
+                               "// Section: helpers.\n"
+                               "\n"
+                               "func Reset() {}\n",
+                               CBM_LANG_GO, "t", "knob.go");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Function", "Reset");
+    ASSERT_NOT_NULL(d);
+    ASSERT_NULL(d->docstring);
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_python_comment_run_and_detached) {
+    CBMFileResult *r = extract("# Turns the knob.\n"
+                               "# One step per call.\n"
+                               "def turn():\n"
+                               "    pass\n"
+                               "\n"
+                               "# Section: helpers.\n"
+                               "\n"
+                               "def reset():\n"
+                               "    pass\n",
+                               CBM_LANG_PYTHON, "t", "knob.py");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *turn = doc_def(r, "Function", "turn");
+    ASSERT_NOT_NULL(turn);
+    ASSERT_STR_EQ(turn->docstring, "# Turns the knob.\n# One step per call.");
+    const CBMDefinition *reset = doc_def(r, "Function", "reset");
+    ASSERT_NOT_NULL(reset);
+    ASSERT_NULL(reset->docstring);
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_rust_attribute_impl_method_and_inner_doc) {
+    CBMFileResult *r = extract("//! The knob crate.\n"
+                               "\n"
+                               "/// Adds two steps.\n"
+                               "/// Wraps on overflow.\n"
+                               "#[inline]\n"
+                               "pub fn add(a: u8, b: u8) -> u8 { a.wrapping_add(b) }\n"
+                               "\n"
+                               "pub struct Knob;\n"
+                               "\n"
+                               "impl Knob {\n"
+                               "    /// Turns the knob once.\n"
+                               "    pub fn turn(&self) {}\n"
+                               "}\n",
+                               CBM_LANG_RUST, "t", "lib.rs");
+    ASSERT_NOT_NULL(r);
+    /* The doc sits above the attribute; a run is joined line by line. */
+    const CBMDefinition *add = doc_def(r, "Function", "add");
+    ASSERT_NOT_NULL(add);
+    ASSERT_STR_EQ(add->docstring, "/// Adds two steps.\n/// Wraps on overflow.");
+    /* impl methods get their doc too; a single comment keeps its bytes. */
+    const CBMDefinition *turn = doc_def(r, "Method", "turn");
+    ASSERT_NOT_NULL(turn);
+    ASSERT_STR_EQ(turn->docstring, "/// Turns the knob once.\n");
+    /* The //! inner doc documents the crate (the File node), not the struct. */
+    const CBMDefinition *knob = doc_def(r, "Struct", "Knob");
+    ASSERT_NOT_NULL(knob);
+    ASSERT_NULL(knob->docstring);
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_ts_export_and_arrow_wrapper) {
+    CBMFileResult *r = extract("/** Exported helper. */\n"
+                               "export function helper(): number {\n"
+                               "    return 1;\n"
+                               "}\n"
+                               "\n"
+                               "/** Arrow helper. */\n"
+                               "export const arrow = (): number => 2;\n",
+                               CBM_LANG_TYPESCRIPT, "t", "knob.ts");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *helper = doc_def(r, "Function", "helper");
+    ASSERT_NOT_NULL(helper);
+    ASSERT_STR_EQ(helper->docstring, "/** Exported helper. */");
+    const CBMDefinition *arrow = doc_def(r, "Function", "arrow");
+    ASSERT_NOT_NULL(arrow);
+    ASSERT_STR_EQ(arrow->docstring, "/** Arrow helper. */");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_csharp_xml_doc_run_and_top_level_function) {
+    CBMFileResult *r = extract("/// Adds one.\n"
+                               "static int AddOne(int x) => x + 1;\n"
+                               "\n"
+                               "namespace Knobs {\n"
+                               "    public class Knob {\n"
+                               "        /// <summary>\n"
+                               "        /// Turns the knob.\n"
+                               "        /// </summary>\n"
+                               "        public void Turn() {}\n"
+                               "    }\n"
+                               "}\n",
+                               CBM_LANG_CSHARP, "t", "Knob.cs");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *turn = doc_def(r, "Method", "Turn");
+    ASSERT_NOT_NULL(turn);
+    ASSERT_STR_EQ(turn->docstring, "/// <summary>\n/// Turns the knob.\n/// </summary>");
+    const CBMDefinition *add = doc_def(r, "Function", "AddOne");
+    ASSERT_NOT_NULL(add);
+    ASSERT_STR_EQ(add->docstring, "/// Adds one.");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_java_javadoc_over_line_comment_and_field) {
+    CBMFileResult *r = extract("class Knob {\n"
+                               "    /** Turns taken so far. */\n"
+                               "    private int steps;\n"
+                               "\n"
+                               "    /** Turns the knob. */\n"
+                               "    // NOSONAR\n"
+                               "    void turn() {}\n"
+                               "}\n",
+                               CBM_LANG_JAVA, "t", "Knob.java");
+    ASSERT_NOT_NULL(r);
+    /* A plain comment between the Javadoc and the method does not replace it. */
+    const CBMDefinition *turn = doc_def(r, "Method", "turn");
+    ASSERT_NOT_NULL(turn);
+    ASSERT_STR_EQ(turn->docstring, "/** Turns the knob. */");
+    const CBMDefinition *steps = doc_def(r, "Field", "steps");
+    ASSERT_NOT_NULL(steps);
+    ASSERT_STR_EQ(steps->docstring, "/** Turns taken so far. */");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_kotlin_kdoc_folded_after_imports) {
+    /* tree-sitter-kotlin folds this KDoc into the node before the class, so it
+     * is no sibling of it. */
+    CBMFileResult *r = extract("package knobs\n"
+                               "\n"
+                               "import knobs.core.Step\n"
+                               "import knobs.core.Turn\n"
+                               "\n"
+                               "/** Base class of every knob. */\n"
+                               "abstract class BaseKnob(step: Step) : Turn(step)\n",
+                               CBM_LANG_KOTLIN, "t", "BaseKnob.kt");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Class", "BaseKnob");
+    ASSERT_NOT_NULL(d);
+    ASSERT_STR_EQ(d->docstring, "/** Base class of every knob. */");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_c_typedef_struct) {
+    CBMFileResult *r = extract("/* A file event. */\n"
+                               "typedef struct aeFileEvent {\n"
+                               "    int mask;\n"
+                               "} aeFileEvent;\n",
+                               CBM_LANG_C, "t", "ae.h");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Class", "aeFileEvent");
+    ASSERT_NOT_NULL(d);
+    ASSERT_STR_EQ(d->docstring, "/* A file event. */");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_perl_pod_section_by_sub_name) {
+    CBMFileResult *r = extract("package Knob;\n"
+                               "\n"
+                               "sub turn { return 1; }\n"
+                               "\n"
+                               "1;\n"
+                               "__END__\n"
+                               "\n"
+                               "=head1 METHODS\n"
+                               "\n"
+                               "=head2 turn\n"
+                               "\n"
+                               "Turns the knob once.\n"
+                               "\n"
+                               "=cut\n",
+                               CBM_LANG_PERL, "t", "Knob.pm");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Function", "turn");
+    ASSERT_NOT_NULL(d);
+    ASSERT_STR_EQ(d->docstring, "=head2 turn\n\nTurns the knob once.");
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_long_comment_kept_whole) {
+    /* One 12-line block comment well past the old 500-byte cut, kept byte for
+     * byte. */
+    char src[4096];
+    char want[2048];
+    int w = snprintf(want, sizeof(want), "/*");
+    for (int i = 0; i < 12; i++) {
+        w += snprintf(want + w, sizeof(want) - (size_t)w,
+                      "\n * Line %02d of a long doc that keeps going past the cut.", i);
+    }
+    snprintf(want + w, sizeof(want) - (size_t)w, "\n */");
+    snprintf(src, sizeof(src), "package knob\n\n%s\nfunc Long() {}\n", want);
+    ASSERT_GT((int)strlen(want), 500);
+
+    CBMFileResult *r = extract(src, CBM_LANG_GO, "t", "knob.go");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *d = doc_def(r, "Function", "Long");
+    ASSERT_NOT_NULL(d);
+    ASSERT_STR_EQ(d->docstring, want);
+    cbm_free_result(r);
+    PASS();
+}
+
+TEST(doc_divider_only_run_is_not_doc) {
+    CBMFileResult *r = extract("/*****************************/\n"
+                               "void knob(void) {}\n"
+                               "\n"
+                               "// \xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\n"
+                               "void dial(void) {}\n"
+                               "\n"
+                               "/* Resets the dial. */\n"
+                               "void reset(void) {}\n",
+                               CBM_LANG_C, "t", "knob.c");
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *knob = doc_def(r, "Function", "knob");
+    ASSERT_NOT_NULL(knob);
+    ASSERT_NULL(knob->docstring);
+    const CBMDefinition *dial = doc_def(r, "Function", "dial"); /* a box-drawing banner */
+    ASSERT_NOT_NULL(dial);
+    ASSERT_NULL(dial->docstring);
+    const CBMDefinition *reset = doc_def(r, "Function", "reset");
+    ASSERT_NOT_NULL(reset);
+    ASSERT_STR_EQ(reset->docstring, "/* Resets the dial. */");
+    cbm_free_result(r);
+    PASS();
+}
+
 /* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file
  * whose root node has hundreds of thousands of FLAT SIBLINGS (580k ////
  * comment lines in the 3.5 MB fourslash fixture) hung extraction for over
@@ -9348,6 +9640,19 @@ SUITE(extraction) {
     RUN_TEST(extract_c_test_dir_marks_is_test_issue1294);
     RUN_TEST(extract_python_method_test_dir_marks_is_test_issue1294);
     RUN_TEST(docstring_utf8_truncation_boundary_issue1017);
+    RUN_TEST(doc_go_comment_group_without_directives);
+    RUN_TEST(doc_go_struct_field);
+    RUN_TEST(doc_go_detached_comment_is_not_doc);
+    RUN_TEST(doc_python_comment_run_and_detached);
+    RUN_TEST(doc_rust_attribute_impl_method_and_inner_doc);
+    RUN_TEST(doc_ts_export_and_arrow_wrapper);
+    RUN_TEST(doc_csharp_xml_doc_run_and_top_level_function);
+    RUN_TEST(doc_java_javadoc_over_line_comment_and_field);
+    RUN_TEST(doc_kotlin_kdoc_folded_after_imports);
+    RUN_TEST(doc_c_typedef_struct);
+    RUN_TEST(doc_perl_pod_section_by_sub_name);
+    RUN_TEST(doc_long_comment_kept_whole);
+    RUN_TEST(doc_divider_only_run_is_not_doc);
     RUN_TEST(extract_ts_decorators_survive_interleaved_comment);
 
     /* #518/#519 — prose carried into docstring so nodes_fts can index it */

tests/test_pipeline.c

@@ -10317,6 +10317,83 @@ TEST(pipeline_docstring_go_class) {
     PASS();
 }
 
+/* Index one file; true when the node with this label and name stores `want` in
+ * its properties JSON. */
+static bool doc_props_contain(const char *file, const char *content, const char *label,
+                              const char *name, const char *want) {
+    const char *files[] = {file};
+    const char *contents[] = {content};
+    if (setup_lang_repo(files, contents, 1) != 0) {
+        return false;
+    }
+    char db[512];
+    snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir);
+    cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL);
+    bool found = false;
+    if (p && cbm_pipeline_run(p) == 0) {
+        cbm_store_t *s = cbm_store_open_path(db);
+        cbm_node_t *nodes = NULL;
+        int nc = 0;
+        if (s && cbm_store_find_nodes_by_label(s, cbm_pipeline_project_name(p), label, &nodes,
+                                               &nc) == 0) {
+            for (int i = 0; i < nc; i++) {
+                if (nodes[i].name && strcmp(nodes[i].name, name) == 0 && nodes[i].properties_json &&
+                    strstr(nodes[i].properties_json, want)) {
+                    found = true;
+                }
+            }
+            cbm_store_free_nodes(nodes, nc);
+        }
+        if (s) {
+            cbm_store_close(s);
+        }
+    }
+    if (p) {
+        cbm_pipeline_free(p);
+    }
+    teardown_lang_repo();
+    return found;
+}
+
+TEST(pipeline_doc_go_package_comment_on_file_node) {
+    /* go/doc: the comment group touching `package` documents the package; it is
+     * stored on the File node. */
+    ASSERT_TRUE(doc_props_contain("knob.go",
+                                  "// Package knob turns knobs.\n"
+                                  "package knob\n\n"
+                                  "func Turn() {}\n",
+                                  "File", "knob.go",
+                                  "\"docstring\":\"// Package knob turns knobs.\""));
+    PASS();
+}
+
+TEST(pipeline_doc_rust_inner_doc_on_file_node) {
+    ASSERT_TRUE(doc_props_contain("lib.rs",
+                                  "//! Knob crate.\n"
+                                  "//! Turns knobs.\n"
+                                  "\n"
+                                  "pub fn turn() {}\n",
+                                  "File", "lib.rs",
+                                  "\"docstring\":\"//! Knob crate.\\n//! Turns knobs.\""));
+    PASS();
+}
+
+TEST(pipeline_doc_long_docstring_stored_whole) {
+    /* A 30-line doc (longer than the old fixed 2 KB properties buffer on its
+     * own) is stored whole, and the fields after it are not dropped. */
+    char src[4096];
+    int s = snprintf(src, sizeof(src), "package knob\n\n");
+    for (int i = 0; i < 30; i++) {
+        s += snprintf(src + s, sizeof(src) - (size_t)s,
+                      "// Line %02d of a long doc comment that keeps on going.\n", i);
+    }
+    snprintf(src + s, sizeof(src) - (size_t)s, "func Long(steps int) int { return steps }\n");
+    ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long",
+                                  "that keeps on going.\\n// Line 29 of a long doc"));
+    ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long", "\"signature\":"));
+    PASS();
+}
+
 TEST(project_name_from_path) {
     /* Port of TestProjectNameFromPath — more cases than integ_pipeline_project_name */
     struct {
@@ -17547,6 +17624,9 @@ SUITE(pipeline) {
     RUN_TEST(pipeline_docstring_java_method);
     RUN_TEST(pipeline_docstring_kotlin_function);
     RUN_TEST(pipeline_docstring_go_class);
+    RUN_TEST(pipeline_doc_go_package_comment_on_file_node);
+    RUN_TEST(pipeline_doc_rust_inner_doc_on_file_node);
+    RUN_TEST(pipeline_doc_long_docstring_stored_whole);
     /* Project name */
     RUN_TEST(project_name_from_path);
     RUN_TEST(project_name_drive_letter_case_insensitive_issue394);