doc-links(adr): supersedes statements ship as SUPERSEDES edges

A fresh held-out census of ADR supersedes statements (the phrase opens its
line or field and names a record directly) passed: 80 of 81 correct (point
0.988, Wilson 95 % lower bound 0.933; 27 ADR repositories from the same
objective code-search walk, at most 25 links per repository, blind readers,
second-reader agreement 1.0). The family ships; the same words in running
prose (`supersedes_prose`) stay rows.

Its one error was a paragraph wrapped before the phrase:
`[ADR-0240](0240-x.md)` / `supersedes ADR-0134 and ...`, the subject on the
line before. A statement may no longer continue a paragraph: a line that no
list, quote or table mark opens, after a line of running text (not blank, a
heading or a `Key: value` field), is prose unless the phrase is itself a
field (`Supersedes:`). On the census this moves exactly that link to prose;
the 80 correct ones stay statements and none is added. Tests cover the
wrapped paragraph, consecutive field lines and a field under running text,
each failing with its rule reverted; the ship gate checks a statement edge
and a prose row in production.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

internal/cbm/doc_adr.c

@@ -38,15 +38,16 @@
 #include <string.h>
 
 enum {
-    ADR_ID_DIGITS = 5,     /* an ADR number has at most this many digits */
-    ADR_PADDED_DIGITS = 3, /* a bare number names a record when padded: `0259`, `012` */
-    ADR_PREFIX_MIN = 2,    /* `DEC-5`: a prefix of 2 ... */
-    ADR_PREFIX_MAX = 12,   /* ... to 12 letters */
-    ADR_DOC_MAX = 500,     /* the node's docstring: the decision, collapsed */
-    ADR_FIELD_MAX = 256,   /* a fact value is cut at a character boundary */
-    ADR_FM_SCAN = 400,     /* front matter ends within this many lines */
-    ADR_HEAD_SCAN = 40,    /* header fields within this many lines when there is no heading */
-    ADR_NAME_MAX = 64,     /* canonical id buffer */
+    ADR_ID_DIGITS = 5,      /* an ADR number has at most this many digits */
+    ADR_PADDED_DIGITS = 3,  /* a bare number names a record when padded: `0259`, `012` */
+    ADR_FIELD_KEY_MAX = 32, /* a field line key: `Status`, `Decision makers` */
+    ADR_PREFIX_MIN = 2,     /* `DEC-5`: a prefix of 2 ... */
+    ADR_PREFIX_MAX = 12,    /* ... to 12 letters */
+    ADR_DOC_MAX = 500,      /* the node's docstring: the decision, collapsed */
+    ADR_FIELD_MAX = 256,    /* a fact value is cut at a character boundary */
+    ADR_FM_SCAN = 400,      /* front matter ends within this many lines */
+    ADR_HEAD_SCAN = 40,     /* header fields within this many lines when there is no heading */
+    ADR_NAME_MAX = 64,      /* canonical id buffer */
     ADR_YEAR_MIN = 1900,
     ADR_YEAR_MAX = 2099,
     ADR_MONTHS = 12,
@@ -1154,6 +1155,61 @@ static bool adr_reported(const adr_doc_t *d, const adr_line_t *l, int i) {
 /* A record reference right after the phrase (past a colon, a table bar and
  * emphasis): a link or an ADR id, `Supersedes: [ADR-3](0003-x.md)`,
  * `| Supersedes | ADR-0003 |`. */
+/* A colon right after the phrase (past emphasis): `Supersedes: ADR-3`, a
+ * field, whatever surrounds it. */
+static bool adr_field_value(const char *s, size_t n) {
+    size_t i = 0;
+    while (i < n && (adr_blank(s[i]) || s[i] == '*' || s[i] == '_')) {
+        i++;
+    }
+    return i < n && s[i] == ':';
+}
+
+/* `Status: Accepted`, `- **Date**: 2024-01-02`: a field line. */
+static bool adr_field_line(const adr_line_t *l) {
+    int i = 0;
+    while (i < l->n && (adr_blank(l->s[i]) || l->s[i] == '-' || l->s[i] == '*' || l->s[i] == '+' ||
+                        l->s[i] == '_' || l->s[i] == '|')) {
+        i++;
+    }
+    int w = i;
+    while (i < l->n && (adr_alpha(l->s[i]) || l->s[i] == ' ' || l->s[i] == '-')) {
+        i++;
+    }
+    if (i == w || i - w > ADR_FIELD_KEY_MAX) {
+        return false;
+    }
+    while (i < l->n && (l->s[i] == '*' || l->s[i] == '_' || l->s[i] == ' ')) {
+        i++;
+    }
+    return i < l->n && (l->s[i] == ':' || l->s[i] == '|');
+}
+
+/* Line k continues a paragraph: no list, quote or table mark opens it and the
+ * line before is running text, not blank, a heading or a field
+ * (`[ADR-0240](0240-x.md)` / `supersedes ADR-0134 and ...`: in a held-out
+ * census the subject was the record on the line before). */
+static bool adr_continuation(const adr_doc_t *d, int k) {
+    const adr_line_t *l = &d->L.v[k];
+    int j = 0;
+    while (j < l->n && adr_blank(l->s[j])) {
+        j++;
+    }
+    if (j < l->n && (l->s[j] == '-' || l->s[j] == '*' || l->s[j] == '+' || l->s[j] == '>' ||
+                     l->s[j] == '|' || adr_digit(l->s[j]))) {
+        return false; /* its own list item, quote or table row */
+    }
+    if (k <= d->L.fm_end || k == 0) {
+        return false;
+    }
+    const adr_line_t *p = &d->L.v[k - SKIP_ONE];
+    int a = 0;
+    while (a < p->n && adr_blank(p->s[a])) {
+        a++;
+    }
+    return p->text && a < p->n && p->s[a] != '#' && !adr_field_line(p);
+}
+
 static bool adr_direct_ref(const char *s, size_t n) {
     size_t i = 0;
     while (i < n && (adr_blank(s[i]) || s[i] == ':' || s[i] == '|' || s[i] == '*' || s[i] == '_')) {
@@ -1200,9 +1256,10 @@ static void adr_relations(adr_doc_t *d, const char *adr_qn, char *superseded_by,
                          * 42 correct, statements 44 of 45) */
                         const char *v = l->s + i + pl;
                         size_t vn = (size_t)(l->n - i - (int)pl);
-                        uint16_t syntax = adr_line_head(l, i) && adr_direct_ref(v, vn)
-                                              ? (uint16_t)CBM_DOCLINK_MD_SUPERSEDES
-                                              : (uint16_t)CBM_DOCLINK_MD_SUPERSEDES_PROSE;
+                        bool statement = adr_line_head(l, i) && adr_direct_ref(v, vn) &&
+                                         (adr_field_value(v, vn) || !adr_continuation(d, k));
+                        uint16_t syntax = statement ? (uint16_t)CBM_DOCLINK_MD_SUPERSEDES
+                                                    : (uint16_t)CBM_DOCLINK_MD_SUPERSEDES_PROSE;
                         adr_targets(d, adr_qn, v, vn, (uint32_t)(k + SKIP_ONE), phrases[p].forward,
                                     syntax, superseded_by, cap);
                     }

internal/cbm/doclink.h

@@ -61,14 +61,14 @@
  *             object directives; Markdown code spans naming a file or one
  *             directory alone, split off the Markdown code-path tier after
  *             its held-out failure; ADR supersedes words in running prose,
- *             split off after three held-out censuses failed on them), short
- *             of a new census after held-out misses were fixed (supersedes
- *             statements), or not yet audited for want of
- *             held-out occurrences (reST include, kernel_doc; AsciiDoc
- *             attribute) means false. Markdown code paths without the bare
- *             part, reST literalinclude and code paths, and PDF qualified
- *             names after the owner rule passed supplementary held-out
- *             audits.
+ *             split off after three held-out censuses failed on them), or
+ *             not yet audited for want of held-out occurrences (reST
+ *             include, kernel_doc; AsciiDoc attribute) means false.
+ *             Markdown code paths without the bare part, reST
+ *             literalinclude and code paths, PDF qualified names after the
+ *             owner rule, and ADR supersedes statements (the phrase opens
+ *             its line or field and names a record: 80 of 81 on a fresh
+ *             census) passed supplementary held-out audits.
  *   edge      the type of the edge a resolved reference becomes: MENTIONS
  *             (documentation names code), SUPERSEDES (an ADR states that it
  *             supersedes another). The edge's source is always the
@@ -92,7 +92,7 @@
     X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS")       \
     /* an ADR's own "supersedes" / "replaces" statement, opening its line or field */          \
     /* and naming a record directly (doc_adr.c): ADR -> ADR */                                 \
-    X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, false, "SUPERSEDES")  \
+    X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES")   \
     /* the same words in running prose ("this record supersedes ADR-3") */                     \
     X(CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_LANG_MARKDOWN, "supersedes_prose", false, false,    \
       "SUPERSEDES")                                                                            \

tests/test_doc_links_md.c

@@ -591,6 +591,8 @@ TEST(doc_links_md_adr) {
 
 /* The production gate (doclink.h): families held back by the held-out audit
  * write rows, not edges; the families that passed still write edges. */
+static int md_supersedes_from(const char *db, const char *file, const char *target);
+
 TEST(doc_links_md_ship_gate) {
     for (size_t i = 0; i < sizeof(DM_HELD_FAMILIES) / sizeof(DM_HELD_FAMILIES[0]); i++) {
         ASSERT_FALSE(cbm_doclink_syntax_ships(DM_HELD_FAMILIES[i]));
@@ -601,12 +603,22 @@ TEST(doc_links_md_ship_gate) {
     ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_LITERALINCLUDE));
     ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_CODE_PATH));
     ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_CODE_PATH));
+    ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_SUPERSEDES));
     char tmp[256];
     snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdgate_XXXXXX");
     ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
     md_write_fixture(tmp);
     /* a file name alone, next to its file: it resolves, and its tier holds it */
     th_write_file(TH_PATH(tmp, "pkg/README.md"), "# Pkg\n\nSee `config.go`.\n");
+    /* an ADR's statement ships as SUPERSEDES; the same words in prose are held */
+    th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
+                  "# Base\n\n## Status\n\nSuperseded\n\n## Decision\n\nWe use A.\n");
+    th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
+                  "# Next\n\n## Status\n\nAccepted\n\nSupersedes [ADR-0001](0001-base.md)\n\n"
+                  "## Decision\n\nWe use B.\n");
+    th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"),
+                  "# Last\n\n## Status\n\nAccepted\n\n## Decision\n\n"
+                  "This record also supersedes ADR-0001 for the cache.\n");
     char db[512];
     snprintf(db, sizeof(db), "%s/md.db", tmp);
     ASSERT_EQ(dm_index(tmp, db, NULL), 0);
@@ -623,6 +635,12 @@ TEST(doc_links_md_ship_gate) {
     ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND "
                            "properties LIKE '%\"syntax\":\"bare_path\"%'"),
               0);
+    ASSERT_EQ(md_supersedes_from(db, "0002-next.md", "0001-base.md"), 1);
+    ASSERT_EQ(md_supersedes_from(db, "0003-last.md", "%"), 0);
+    ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
+                           "reason = 'below_bar_tier' AND syntax = 'supersedes_prose' "
+                           "AND rel_path = 'docs/adr/0003-last.md'"),
+              1);
     th_cleanup(tmp);
     PASS();
 }
@@ -719,6 +737,20 @@ TEST(doc_links_md_adr_reported) {
     th_write_file(TH_PATH(tmp, "docs/adr/0010-field.md"),
                   "# Field\n\n- Status: Accepted\n- **Supersedes:** ADR-0002\n\n"
                   "## Decision\n\nWe use G.\n");
+    /* a paragraph wrapped before the phrase: its subject is the line before */
+    th_write_file(
+        TH_PATH(tmp, "docs/adr/0011-wrap.md"),
+        "# Wrap\n\n## Status\n\nAccepted\n\n## Context\n\nThe sealed store of\n"
+        "[ADR 0002](0002-next.md)\nsupersedes ADR-0001 and leaves this record standing.\n\n"
+        "## Decision\n\nWe use I.\n");
+    /* field lines one under the other: each its own statement */
+    th_write_file(TH_PATH(tmp, "docs/adr/0012-fields.md"),
+                  "# Fields\n\nStatus: Accepted\nSupersedes [ADR-0001](0001-base.md)\n\n"
+                  "## Decision\n\nWe use J.\n");
+    /* a `Supersedes:` field right under running text is still a field */
+    th_write_file(TH_PATH(tmp, "docs/adr/0013-colon.md"),
+                  "# Colon\n\n## Status\n\nAccepted after a long review of the store\n"
+                  "Supersedes: [ADR 0002](0002-next.md)\n\n## Decision\n\nWe use K.\n");
     char db[512];
     snprintf(db, sizeof(db), "%s/adr.db", tmp);
     ASSERT_EQ(dm_index(tmp, db, NULL), 0);
@@ -738,6 +770,10 @@ TEST(doc_links_md_adr_reported) {
     ASSERT_EQ(md_supersedes_syntax(db, "0008-tools.md", "supersedes"), 1);
     ASSERT_EQ(md_supersedes_syntax(db, "0009-table.md", "supersedes"), 1);
     ASSERT_EQ(md_supersedes_syntax(db, "0010-field.md", "supersedes"), 1);
+    ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes"), 0);
+    ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes_prose"), 1);
+    ASSERT_EQ(md_supersedes_syntax(db, "0012-fields.md", "supersedes"), 1);
+    ASSERT_EQ(md_supersedes_syntax(db, "0013-colon.md", "supersedes"), 1);
     th_cleanup(tmp);
     PASS();
 }

tests/test_doc_mentions_helpers.h

@@ -416,9 +416,9 @@ static inline int dm_reason_lines(const char *db, const char *block) {
  * so it is the resolution an audit would ship; doc_links_md_ship_gate tests
  * the production gate itself. */
 static const int DM_HELD_FAMILIES[] = {
-    CBM_DOCLINK_MD_BARE_PATH,   CBM_DOCLINK_MD_SUPERSEDES,  CBM_DOCLINK_MD_SUPERSEDES_PROSE,
-    CBM_DOCLINK_RST_OBJECT,     CBM_DOCLINK_RST_INCLUDE,    CBM_DOCLINK_RST_KERNEL_DOC,
-    CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, CBM_DOCLINK_ADOC_CODE_NAME,
+    CBM_DOCLINK_MD_BARE_PATH,   CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_DOCLINK_RST_OBJECT,
+    CBM_DOCLINK_RST_INCLUDE,    CBM_DOCLINK_RST_KERNEL_DOC,      CBM_DOCLINK_ADOC_ATTRIBUTE,
+    CBM_DOCLINK_ADOC_CODE_PATH, CBM_DOCLINK_ADOC_CODE_NAME,
 };
 
 static inline void dm_ship_held_families(void) {