fix(doc-links): a member the code file writes keeps its file edge (#2587)

A `path::member` reference became a missing row whenever the member
lookup found no definition node, but real documentation names members
the file has without a node the lookup binds: a call form
(`src/util.py::clamp()`), a C macro or a prototype in a header, a
TypeScript namespace, a Python enum member, an annotated class
attribute. Those lost their correct file edge and were counted as doc
rot.

A member now reads without a trailing call form, so `clamp()` binds
`clamp`. When no definition matches, the resolver reads the code file
and calls the member missing only when the file does not write its name
at all (a whole identifier, case-sensitive); a member the file writes
keeps the file edge it had before. A file that cannot be read whole
makes no claim. Ambiguous members are unchanged.

Test: doc_links_md_member_written_issue2587 indexes each shape under its
own heading, and two members that stay missing (Color::BLUE, which the
file writes only as LIGHT_BLUE and a lowercase "blue", and
no_such_function). On the previous head all seven shapes are missing
rows; without the call-form trim the call form is; without the
identifier boundary, or with a case-insensitive match, BLUE takes the
file edge.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/pipeline/doc_links_md.c

@@ -10,15 +10,17 @@
  * document says (the field tests' rule for bare file names).
  *
  * Into the file: a line range picks the innermost code definition that holds
- * the WHOLE range; a member (`path::Name`, `path::Owner::name`) picks the one
- * definition of that name in the file. Otherwise the edge goes to the file,
- * and the range stays on it ("target_lines"). Every edge is exact: the
- * document wrote the path.
+ * the WHOLE range; a member (`path::Name`, `path::Owner::name`; a call form
+ * `path::name()` reads as `path::name`) picks the one definition of that name
+ * in the file. Otherwise the edge goes to the file, and the range stays on it
+ * ("target_lines"); so does a member the code file writes without a
+ * definition the graph binds (a macro, a prototype, an enum member, a class
+ * attribute). Every edge is exact: the document wrote the path.
  *
  * Unresolved:
  *   missing    the path is not in the graph, but its directory is: the
  *              document names something that is not there (doc rot); or a
- *              code file's member that the file does not define
+ *              member whose name the code file does not write at all
  *   ambiguous  a file name alone that is not next to the document but exists
  *              elsewhere in the repository: the document does not say which;
  *              or a member name the file defines more than once, with no
@@ -36,11 +38,14 @@
 #include "discover/discover.h" /* cbm_language_for_filename */
 #include "doclink.h"
 #include "foundation/arena.h"
+#include "foundation/compat_fs.h" /* cbm_fopen */
 #include "foundation/constants.h"
 #include "foundation/hash_table.h"
+#include "foundation/limits.h" /* cbm_max_file_bytes */
 #include "foundation/mem_core.h"
 #include "graph_buffer/graph_buffer.h"
 
+#include <ctype.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -91,6 +96,7 @@ typedef struct {
 typedef struct {
     const char *project;
     size_t project_len;
+    const char *repo;       /* the repository root (NULL: unknown; no file is read) */
     const char **run_paths; /* run_file -> rel_path (NULL: not a Markdown file of this run) */
     int run_count;
     /* base name -> number of File nodes with it (a file name alone that is
@@ -288,6 +294,7 @@ static void *mdr_build(const cbm_doclink_build_in_t *in) {
     cbm_arena_init(&x->arena);
     x->project = in->ctx ? in->ctx->project_name : NULL;
     x->project_len = x->project ? strlen(x->project) : 0;
+    x->repo = in->ctx ? in->ctx->repo_path : NULL;
     x->run_count = in->run_file_count;
     x->run_paths = in->run_file_count > 0
                        ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER,
@@ -420,14 +427,31 @@ static bool mdr_code_label(const char *label) {
     return false;
 }
 
+/* The length of the member text as a name: a call form (`name()`,
+ * `Owner.name(a, b)`) ends before its first '('. */
+static size_t mdr_member_len(const char *member) {
+    size_t n = strlen(member);
+    if (n > 0 && member[n - SKIP_ONE] == ')') {
+        const char *open = memchr(member, '(', n);
+        if (open) {
+            return (size_t)(open - member);
+        }
+    }
+    return n;
+}
+
 /* The member text's last segment (after "::" or "."), and the one before it. */
 static void mdr_member_names(const char *member, const char **name, size_t *name_len,
                              const char **owner, size_t *owner_len) {
     const char *segs[PAIR_LEN] = {NULL, NULL};
     size_t lens[PAIR_LEN] = {0, 0};
     const char *s = member;
-    while (*s) {
+    const char *end = member + mdr_member_len(member);
+    while (s < end) {
         size_t n = strcspn(s, ":.");
+        if (n > (size_t)(end - s)) {
+            n = (size_t)(end - s);
+        }
         if (n > 0) {
             segs[0] = segs[SKIP_ONE];
             lens[0] = lens[SKIP_ONE];
@@ -435,7 +459,7 @@ static void mdr_member_names(const char *member, const char **name, size_t *name
             lens[SKIP_ONE] = n;
         }
         s += n;
-        while (*s == ':' || *s == '.') {
+        while (s < end && (*s == ':' || *s == '.')) {
             s++;
         }
     }
@@ -564,6 +588,63 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path
     return NULL;
 }
 
+/* A byte of an identifier: a letter, a digit, '_', and every byte of a
+ * multi-byte character (pipeline_incremental.c doc_row_ident_byte). */
+static bool mdr_ident_byte(unsigned char c) {
+    return isalnum(c) || c == '_' || c >= CBM_SZ_128;
+}
+
+/* Does text[0, n) hold name[0, len) as a whole identifier (case-sensitive)? */
+static bool mdr_writes_ident(const char *text, size_t n, const char *name, size_t len) {
+    for (size_t i = 0; len > 0 && i + len <= n;) {
+        const char *hit = memchr(text + i, name[0], n - len - i + SKIP_ONE);
+        if (!hit) {
+            return false;
+        }
+        size_t at = (size_t)(hit - text);
+        if (memcmp(hit, name, len) == 0 && (at == 0 || !mdr_ident_byte((unsigned char)hit[-1])) &&
+            (at + len == n || !mdr_ident_byte((unsigned char)hit[len]))) {
+            return true;
+        }
+        i = at + SKIP_ONE;
+    }
+    return false;
+}
+
+/* Does the code file `path` write the member's name at all? A name the graph
+ * holds no definition for (a macro, a prototype, an enum member, a class
+ * attribute) is still there when the file writes it. Read from the file,
+ * never cached (resolvers run on worker threads). True -- no claim -- when
+ * the file cannot be read whole: a member is missing only when the file was
+ * read and does not write its name. */
+static bool mdr_file_writes(const mdr_index_t *x, const char *path, const char *member) {
+    const char *name;
+    const char *owner;
+    size_t name_len;
+    size_t owner_len;
+    mdr_member_names(member, &name, &name_len, &owner, &owner_len);
+    char abs_path[CBM_PATH_MAX];
+    int w = name && x->repo ? snprintf(abs_path, sizeof(abs_path), "%s/%s", x->repo, path) : 0;
+    if (w <= 0 || (size_t)w >= sizeof(abs_path)) {
+        return true;
+    }
+    FILE *f = cbm_fopen(abs_path, "rb");
+    if (!f) {
+        return true;
+    }
+    long size = fseek(f, 0, SEEK_END) == 0 ? ftell(f) : CBM_NOT_FOUND;
+    char *text = size >= 0 && size <= cbm_max_file_bytes() && fseek(f, 0, SEEK_SET) == 0
+                     ? (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE)
+                     : NULL;
+    bool writes = true;
+    if (text && fread(text, SKIP_ONE, (size_t)size, f) == (size_t)size) {
+        writes = mdr_writes_ident(text, (size_t)size, name, name_len);
+    }
+    (void)fclose(f);
+    cbm_free(CBM_MEM_CLASS_OTHER, text);
+    return writes;
+}
+
 /* Is `path` of a file type the indexer reads? */
 static bool mdr_indexable(const char *path) {
     return cbm_language_for_filename(mdr_base(path)) != CBM_LANG_COUNT;
@@ -1181,10 +1262,13 @@ static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, cons
                     ? mdr_segment(x, t, r->first, r->last, r->member, &member_result)
                     : NULL;
             /* A named member that a code file does not define is no coarser
-             * link to the file: it is a stale or unclear reference. A
-             * document's own headings are not definitions, so members of
-             * other files keep the file edge. */
-            if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t)) {
+             * link to the file: it is a stale or unclear reference. One the
+             * file writes without a definition the graph binds (a macro, a
+             * prototype, an enum member) is there, and keeps the file edge.
+             * A document's own headings are not definitions, so members of
+             * other files keep the file edge too. */
+            if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t) &&
+                (member_result == MDR_MEMBER_AMBIGUOUS || !mdr_file_writes(x, t, r->member))) {
                 out->kind = CBM_DOCLINK_UNRESOLVED;
                 out->reason = member_result == MDR_MEMBER_AMBIGUOUS ? CBM_DOCLINK_REASON_AMBIGUOUS
                                                                     : CBM_DOCLINK_REASON_MISSING;

tests/test_doc_links_md.c

@@ -455,6 +455,99 @@ TEST(doc_links_md_member_not_defined_issue2587) {
     PASS();
 }
 
+/* MENTIONS edges from the section whose qualified name ends with `section` to
+ * the File node of `file`. */
+static int md_file_edges(const char *db, const char *section, const char *file) {
+    char sql[512];
+    snprintf(sql, sizeof(sql),
+             "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
+             "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
+             "AND s.qualified_name LIKE '%%.%s' AND t.label = 'File' AND t.file_path = '%s'",
+             section, file);
+    return dm_count(db, sql);
+}
+
+/* #2587, the members a code file has although the graph defines no node for
+ * them: a call form, a macro, a prototype, a namespace, an enum member, a
+ * class attribute. Each is there -- the file writes it -- so it keeps its
+ * edge; only a name the file does not write at all is missing. */
+static const struct {
+    const char *section;
+    const char *raw;
+    const char *file;
+    bool missing;
+} MD_WRITTEN[] = {
+    {"CallForm", "src/util.py::clamp()", NULL, false},
+    {"Macro", "include/limits.h::MAX_ITEMS", "include/limits.h", false},
+    {"FunctionMacro", "include/limits.h::SQUARE", "include/limits.h", false},
+    {"Prototype", "include/limits.h::limits_init", "include/limits.h", false},
+    {"Namespace", "src/ns.ts::Shapes", "src/ns.ts", false},
+    {"EnumMember", "src/color.py::Color::RED", "src/color.py", false},
+    {"Attribute", "src/model.py::Model::name", "src/model.py", false},
+    /* not written: `blue` and LIGHT_BLUE are no BLUE */
+    {"NotWritten", "src/color.py::Color::BLUE", "src/color.py", true},
+    {"NotDefined", "src/util.py::no_such_function", "src/util.py", true},
+};
+
+TEST(doc_links_md_member_written_issue2587) {
+    char tmp[256];
+    snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX");
+    ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+    char doc[2048] = "";
+    size_t used = 0;
+    for (size_t i = 0; i < sizeof(MD_WRITTEN) / sizeof(MD_WRITTEN[0]); i++) {
+        used += (size_t)snprintf(doc + used, sizeof(doc) - used, "# %s\n\nSee `%s`.\n\n",
+                                 MD_WRITTEN[i].section, MD_WRITTEN[i].raw);
+        ASSERT(used < sizeof(doc));
+    }
+    th_write_file(TH_PATH(tmp, "docs/written.md"), doc);
+    th_write_file(TH_PATH(tmp, "src/util.py"), "def clamp(x, lo, hi):\n"
+                                               "    return max(lo, min(hi, x))\n");
+    th_write_file(TH_PATH(tmp, "include/limits.h"), "#ifndef LIMITS_H\n"
+                                                    "#define LIMITS_H\n"
+                                                    "#define MAX_ITEMS 64\n"
+                                                    "#define SQUARE(x) ((x) * (x))\n"
+                                                    "int limits_init(void);\n"
+                                                    "#endif\n");
+    th_write_file(TH_PATH(tmp, "src/ns.ts"), "export namespace Shapes {\n"
+                                             "    export const unit = 1;\n"
+                                             "}\n");
+    th_write_file(TH_PATH(tmp, "src/color.py"), "from enum import Enum\n"
+                                                "\n"
+                                                "\n"
+                                                "class Color(Enum):\n"
+                                                "    RED = 1\n"
+                                                "    LIGHT_BLUE = 2  # blue\n");
+    th_write_file(TH_PATH(tmp, "src/model.py"), "class Model:\n"
+                                                "    name: str\n");
+    char db[512];
+    snprintf(db, sizeof(db), "%s/md.db", tmp);
+    ASSERT_EQ(dm_index(tmp, db, NULL), 0);
+    char props[512];
+    char reason[64];
+
+    /* every reference, under its own heading: an edge, or a missing row */
+    int wrong = 0;
+    for (size_t i = 0; i < sizeof(MD_WRITTEN) / sizeof(MD_WRITTEN[0]); i++) {
+        dm_row(db, "docs/written.md", MD_WRITTEN[i].raw, reason, sizeof(reason), NULL, 0);
+        int edges = MD_WRITTEN[i].file
+                        ? md_file_edges(db, MD_WRITTEN[i].section, MD_WRITTEN[i].file)
+                        : dm_mentions_from(db, MD_WRITTEN[i].section);
+        bool ok = MD_WRITTEN[i].missing ? strcmp(reason, "missing") == 0 && edges == 0
+                                        : reason[0] == '\0' && edges >= 1;
+        if (!ok) {
+            printf("  %s: row '%s', %d edge(s)\n", MD_WRITTEN[i].raw, reason, edges);
+            wrong++;
+        }
+    }
+    ASSERT_EQ(wrong, 0);
+    /* the call form binds the function itself */
+    ASSERT_EQ(md_edge(db, "CallForm", "util.clamp", props, sizeof(props)), 1);
+
+    th_cleanup(tmp);
+    PASS();
+}
+
 /* ── incremental == full ─────────────────────────────────────────── */
 
 TEST(doc_links_md_incremental) {
@@ -933,6 +1026,7 @@ SUITE(doc_links_md) {
     RUN_TEST(doc_links_md_classify_span);
     RUN_TEST(doc_links_md_resolve);
     RUN_TEST(doc_links_md_member_not_defined_issue2587);
+    RUN_TEST(doc_links_md_member_written_issue2587);
     RUN_TEST(doc_links_md_incremental);
     RUN_TEST(doc_links_md_adr);
     RUN_TEST(doc_links_md_adr_label_choice);