nprasad2077/codebase-memory-mcp
1
0 Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-04 10:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

doc-links: fixes from the supplementary held-out audits; four more families ship

Browse Source 
Supplementary held-out audits, each registered before its queries and run on
repositories chosen by an objective code-search walk, disjoint from every
earlier audit and development corpus (blind readers, a second reader on
every link not judged correct and on a seeded tenth of the rest; bar: Wilson
95 % lower bound >= 0.90 and point >= 0.95). A fix is judged on a NEW sample,
never on the one that found the error.

Ship now:
- Markdown code_path, refined to code spans with two or more path segments:
  97 of 100 (lower bound 0.916). A file or one directory named alone in a
  code span is its own family, bare_path, held: across the judged samples it
  was 28 of 41 correct (`config.yaml` the reader's own file, `doc/` another
  project's directory). A span without a word character (`../`, `./`) is no
  path at all.
- reST literalinclude: 100 of 100; an open range (`:lines: 5-`, `-5`) now
  reads the included file for its end and binds like a closed one.
- reST code_path: 99 of 100; its one error, a bare `../`, is fixed above.
- PDF qualified names: 97 of 99 after the owner rule: a definition whose
  qualified name lacks its owner (a Go method under its package) matches
  only owner-qualified text, so `time.Now` no longer binds a receiver's Now.

Fixed, still held:
- reST object directives: a function or class directive never binds a
  Module, Folder or File (an example script's module took them); a new
  sample is due.
- ADR supersedes: three censuses failed (37/38, 144/163, 82/87). Fixed from
  them: a combined template label (`Supersedes / Depends on:`), a value of
  "none" or "n/a" followed by records it extends, another record as the
  subject (`ADR 0239 supersedes ...`, a table cell `0259 (supersedes ...)`),
  a full stop after a number ending the target sentence while a
  `[text](destination)` link stays one reference (adr-tools'
  `Supersedes [3. Title](0003-...md)`). The family is split: a statement that
  opens its line or field and names a record directly stays `supersedes`
  (44 of 45 on the last census); the same words in running prose are
  `supersedes_prose` (38 of 42), held. Both still held in this commit; a
  fresh census of statements passed afterwards and is acted on next.
- An ADR log in a repository without code definitions lost every supersedes
  link by file: the ADR table was filled after an early return. It is now
  filled first.

Every fix has a test that fails with the fix reverted.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-10-09 02:55:23 +02:00
1 parent e1dfdab5cb
commit 708569df22
10 files changed
+598 -126
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/cbm/doc_adr.c
+145 -22
View File
@@ -38,14 +38,15 @@
#include <string.h>
enum {
ADR_ID_DIGITS = 5, /* an ADR number has at most this many digits */
ADR_PREFIX_MIN = 2, /* `DEC-5`: a prefix of 2 ... */
ADR_PREFIX_MAX = 12, /* ... to 12 letters */
ADR_DOC_MAX = 500, /* the node's docstring: the decision, collapsed */
ADR_FIELD_MAX = 256, /* a fact value is cut at a character boundary */
ADR_FM_SCAN = 400, /* front matter ends within this many lines */
ADR_HEAD_SCAN = 40, /* header fields within this many lines when there is no heading */
ADR_NAME_MAX = 64, /* canonical id buffer */
ADR_ID_DIGITS = 5, /* an ADR number has at most this many digits */
ADR_PADDED_DIGITS = 3, /* a bare number names a record when padded: `0259`, `012` */
ADR_PREFIX_MIN = 2, /* `DEC-5`: a prefix of 2 ... */
ADR_PREFIX_MAX = 12, /* ... to 12 letters */
ADR_DOC_MAX = 500, /* the node's docstring: the decision, collapsed */
ADR_FIELD_MAX = 256, /* a fact value is cut at a character boundary */
ADR_FM_SCAN = 400, /* front matter ends within this many lines */
ADR_HEAD_SCAN = 40, /* header fields within this many lines when there is no heading */
ADR_NAME_MAX = 64, /* canonical id buffer */
ADR_YEAR_MIN = 1900,
ADR_YEAR_MAX = 2099,
ADR_MONTHS = 12,
@@ -869,7 +870,9 @@ typedef struct {
adr_lines_t L;
adr_head_t *heads;
int nheads;
int title_head; /* index into heads, or -1 */
int title_head; /* index into heads, or -1 */
bool numbered; /* the file name gives the record a number: */
unsigned number; /* this one */
} adr_doc_t;
/* The paragraph after heading h (its first non-blank text lines). */
@@ -969,7 +972,7 @@ static bool adr_id_ref(const char *s, size_t n, unsigned *number, size_t *len) {
}
static void adr_push_relation(adr_doc_t *d, const char *adr_qn, const char *raw, size_t n,
uint32_t line) {
uint32_t line, uint16_t syntax) {
CBMArena *a = d->ctx->arena;
char *text = cbm_arena_strndup(a, raw, n);
if (!text) {
@@ -980,21 +983,33 @@ static void adr_push_relation(adr_doc_t *d, const char *adr_qn, const char *raw,
.raw = text,
.line = line,
.def_line = SKIP_ONE,
.syntax = (uint16_t)CBM_DOCLINK_MD_SUPERSEDES,
.syntax = syntax,
.flags = 0};
cbm_doclinks_push(&d->ctx->result->doc_links, a, link);
}
/* The targets after a relation phrase, up to the end of its sentence: link
* destinations and ADR ids. Forward relations become tokens; for a backward
* one ("superseded by") the targets are joined into `*joined`. */
* destinations and ADR ids. Forward relations become tokens of family
* `syntax`; for a backward one ("superseded by") the targets are joined into
* `*joined`. */
static void adr_targets(adr_doc_t *d, const char *adr_qn, const char *s, size_t n, uint32_t line,
bool forward, char *joined, size_t jcap) {
bool forward, uint16_t syntax, char *joined, size_t jcap) {
size_t end = n < ADR_RELATION_SPAN ? n : ADR_RELATION_SPAN;
for (size_t i = 0; i + SKIP_ONE < end; i++) {
if ((s[i] == '.' && (i + SKIP_ONE == end || adr_blank(s[i + SKIP_ONE])) &&
!(i > 0 && adr_digit(s[i - SKIP_ONE]))) ||
s[i] == ';') {
/* a link is one reference whatever its text holds (adr-tools writes
* `Supersedes [3. Show links](0003-show-links.md)`) */
const char *rb = s[i] == '[' ? memchr(s + i, ']', end - i) : NULL;
const char *close = rb && rb + SKIP_ONE < s + end && rb[SKIP_ONE] == '('
? memchr(rb, ')', (size_t)(s + end - rb))
: NULL;
if (close) {
i = (size_t)(close - s);
continue;
}
/* a full stop after a number ends the sentence too (`supersedes 0092
* and 0094. ADR 0202 is ...`: in a held-out audit the next sentence's
* record was taken for a target) */
if ((s[i] == '.' && (i + SKIP_ONE == end || adr_blank(s[i + SKIP_ONE]))) || s[i] == ';') {
end = i;
break;
}
@@ -1026,7 +1041,7 @@ static void adr_targets(adr_doc_t *d, const char *adr_qn, const char *s, size_t
continue;
}
if (forward) {
adr_push_relation(d, adr_qn, target, tn, line);
adr_push_relation(d, adr_qn, target, tn, line, syntax);
} else if (joined) {
size_t w = strlen(joined);
bool seen = false;
@@ -1058,6 +1073,97 @@ static bool adr_line_head(const adr_line_t *l, int i) {
return true;
}
/* A relation word joined to another by a slash, `Supersedes / Depends on:`:
* a template's combined field label, which states neither relation (in a
* held-out audit the record depended on the one it named). */
static bool adr_label_choice(const adr_line_t *l, int a, int b) {
while (b < l->n && (l->s[b] == ' ' || l->s[b] == '\t')) {
b++;
}
while (a > 0 && (l->s[a - SKIP_ONE] == ' ' || l->s[a - SKIP_ONE] == '\t')) {
a--;
}
return (b < l->n && l->s[b] == '/') || (a > 0 && l->s[a - SKIP_ONE] == '/');
}
/* The field's value is "none" or "n/a" (`Supersedes: none (it extends ADR
* 0031)`): the record states no relation, and the records it goes on to name
* are extended or refined (a held-out audit's most frequent error). */
static bool adr_none_value(const char *s, size_t n) {
size_t i = 0;
while (i < n && (adr_blank(s[i]) || s[i] == ':' || s[i] == '*' || s[i] == '_')) {
i++;
}
static const char *const none[] = {"none", "n/a", NULL};
for (size_t k = 0; none[k]; k++) {
size_t w = strlen(none[k]);
if (adr_starts_ci(s + i, n - i, none[k]) && (i + w == n || !adr_alpha(s[i + w]))) {
return true;
}
}
return false;
}
/* Another record is the phrase's subject (`ADR 0239 supersedes ADR 0203`, a
* table cell `0259 (supersedes ADR 0114's ...)`, a link to a record before
* it): this record reports that record's relation and states none of its
* own. Its own number as the subject is its own statement. */
static bool adr_reported(const adr_doc_t *d, const adr_line_t *l, int i) {
int k = i;
while (k > 0 &&
(adr_blank(l->s[k - SKIP_ONE]) || l->s[k - SKIP_ONE] == '(' ||
l->s[k - SKIP_ONE] == '*' || l->s[k - SKIP_ONE] == '_' || l->s[k - SKIP_ONE] == '`')) {
k--;
}
if (k >= PAIR_LEN && l->s[k - SKIP_ONE] == ')') {
for (int o = k - PAIR_LEN; o > 0; o--) { /* `[text](destination)` */
if (l->s[o] == '(') {
return l->s[o - SKIP_ONE] == ']';
}
if (l->s[o] == ')' || adr_blank(l->s[o])) {
break;
}
}
return false;
}
int e = k;
while (k > 0 && adr_digit(l->s[k - SKIP_ONE])) {
k--;
}
if (e == k || (k > 0 && adr_alpha(l->s[k - SKIP_ONE]))) {
return false;
}
size_t used = 0;
unsigned v = 0;
if (!adr_number_at(l->s + k, (size_t)(e - k), &used, &v)) {
return false;
}
unsigned ref = 0;
size_t len = 0;
bool id = k >= PAIR_LEN * PAIR_LEN &&
adr_id_ref(l->s + k - PAIR_LEN * PAIR_LEN, (size_t)(e - k) + PAIR_LEN * PAIR_LEN,
&ref, &len) &&
len == (size_t)(e - k) + PAIR_LEN * PAIR_LEN;
/* a bare number is a record's only when padded like one (`0259`) */
if (!id && e - k < ADR_PADDED_DIGITS) {
return false;
}
return !(d->numbered && v == d->number);
}
/* A record reference right after the phrase (past a colon, a table bar and
* emphasis): a link or an ADR id, `Supersedes: [ADR-3](0003-x.md)`,
* `| Supersedes | ADR-0003 |`. */
static bool adr_direct_ref(const char *s, size_t n) {
size_t i = 0;
while (i < n && (adr_blank(s[i]) || s[i] == ':' || s[i] == '|' || s[i] == '*' || s[i] == '_')) {
i++;
}
unsigned number = 0;
size_t len = 0;
return i < n && (s[i] == '[' || adr_id_ref(s + i, n - i, &number, &len));
}
/* "supersedes" and "superseded by" are ADR words wherever they stand;
* "replaces" and "replaced by" are ordinary verbs of technical prose ("this
* middleware replaces the recovery described in ADR-22") and state a relation
@@ -1084,9 +1190,22 @@ static void adr_relations(adr_doc_t *d, const char *adr_qn, char *superseded_by,
size_t pl = strlen(phrases[p].phrase);
if (adr_starts_ci(l->s + i, (size_t)(l->n - i), phrases[p].phrase) &&
(i + (int)pl == l->n || !adr_alpha(l->s[i + (int)pl])) &&
(!phrases[p].line_head || adr_line_head(l, i))) {
adr_targets(d, adr_qn, l->s + i + pl, (size_t)(l->n - i - (int)pl),
(uint32_t)(k + SKIP_ONE), phrases[p].forward, superseded_by, cap);
(!phrases[p].line_head || adr_line_head(l, i)) &&
!adr_label_choice(l, i, i + (int)pl)) {
if (!adr_none_value(l->s + i + pl, (size_t)(l->n - i - (int)pl)) &&
!adr_reported(d, l, i)) {
/* a statement opens its line or field and names a
* record directly; the same words in running prose
* are their own family (held-out audits: prose 38 of
* 42 correct, statements 44 of 45) */
const char *v = l->s + i + pl;
size_t vn = (size_t)(l->n - i - (int)pl);
uint16_t syntax = adr_line_head(l, i) && adr_direct_ref(v, vn)
? (uint16_t)CBM_DOCLINK_MD_SUPERSEDES
: (uint16_t)CBM_DOCLINK_MD_SUPERSEDES_PROSE;
adr_targets(d, adr_qn, v, vn, (uint32_t)(k + SKIP_ONE), phrases[p].forward,
syntax, superseded_by, cap);
}
i += (int)pl - SKIP_ONE;
break;
}
@@ -1177,7 +1296,11 @@ void cbm_adr_extract(CBMExtractCtx *ctx) {
}
CBMArena *scratch = ctx->scratch ? ctx->scratch : ctx->arena;
CBMArena *a = ctx->arena;
adr_doc_t d = {.ctx = ctx, .title_head = CBM_NOT_FOUND};
adr_doc_t d = {.ctx = ctx,
.title_head = CBM_NOT_FOUND,
.numbered = id.kind == ADR_ID_ADR || id.kind == ADR_ID_NUMBER ||
id.kind == ADR_ID_PREFIX,
.number = id.number};
if (!adr_split_lines(scratch, ctx->source, ctx->source_len, &d.L)) {
ctx->result->doc_links.failed = true;
return;
internal/cbm/doclink.h
+52 -38
View File
@@ -57,10 +57,18 @@
* row with reason below_bar_tier (an unresolved one keeps its own
* reason). Nothing else about the family changes. A document
* family ships once a held-out audit passed it: below the bar
* (pdf_qn; Markdown and AsciiDoc code spans naming paths, AsciiDoc
* code names, reST code paths, supersedes) or not yet audited for
* want of held-out occurrences (reST object, include,
* literalinclude, kernel_doc; AsciiDoc attribute) means false.
* (AsciiDoc code spans naming paths, AsciiDoc code names, reST
* object directives; Markdown code spans naming a file or one
* directory alone, split off the Markdown code-path tier after
* its held-out failure; ADR supersedes words in running prose,
* split off after three held-out censuses failed on them), short
* of a new census after held-out misses were fixed (supersedes
* statements), or not yet audited for want of
* held-out occurrences (reST include, kernel_doc; AsciiDoc
* attribute) means false. Markdown code paths without the bare
* part, reST literalinclude and code paths, and PDF qualified
* names after the owner rule passed supplementary held-out
* audits.
* edge the type of the edge a resolved reference becomes: MENTIONS
* (documentation names code), SUPERSEDES (an ADR states that it
* supersedes another). The edge's source is always the
@@ -69,40 +77,46 @@
*
* The enum below and the family table (doclink.c) are both generated from this
* list, so a value cannot exist without its name and its gate. */
#define CBM_DOCLINK_FAMILY_LIST(X) \
/* csharp */ \
X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \
/* markdown (doclink_md.c): what a document's text names explicitly */ \
X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \
/* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \
X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, false, "SUPERSEDES") \
/* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \
/* names) only decides which line fragments a cross-line join replaces */ \
X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, false, "MENTIONS") \
X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \
/* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \
X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \
/* asciidoc (doc_adoc.c): includes, Antora API attributes, monospace spans */ \
X(CBM_DOCLINK_ADOC_INCLUDE, CBM_LANG_ASCIIDOC, "include", false, true, "MENTIONS") \
X(CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_LANG_ASCIIDOC, "attribute", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_PATH, CBM_LANG_ASCIIDOC, "code_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_NAME, CBM_LANG_ASCIIDOC, "code_name", false, false, "MENTIONS") \
/* any language (CBM_LANG_COUNT): a URL is never an edge */ \
#define CBM_DOCLINK_FAMILY_LIST(X) \
/* csharp */ \
X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \
/* markdown (doclink_md.c): what a document's text names explicitly */ \
X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \
/* a code span naming a file or directory alone (`config.yaml`, `doc/`) */ \
X(CBM_DOCLINK_MD_BARE_PATH, CBM_LANG_MARKDOWN, "bare_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \
/* an ADR's own "supersedes" / "replaces" statement, opening its line or field */ \
/* and naming a record directly (doc_adr.c): ADR -> ADR */ \
X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, false, "SUPERSEDES") \
/* the same words in running prose ("this record supersedes ADR-3") */ \
X(CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_LANG_MARKDOWN, "supersedes_prose", false, false, \
"SUPERSEDES") \
/* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \
/* names) only decides which line fragments a cross-line join replaces */ \
X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \
/* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \
X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \
/* asciidoc (doc_adoc.c): includes, Antora API attributes, monospace spans */ \
X(CBM_DOCLINK_ADOC_INCLUDE, CBM_LANG_ASCIIDOC, "include", false, true, "MENTIONS") \
X(CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_LANG_ASCIIDOC, "attribute", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_PATH, CBM_LANG_ASCIIDOC, "code_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_NAME, CBM_LANG_ASCIIDOC, "code_name", false, false, "MENTIONS") \
/* any language (CBM_LANG_COUNT): a URL is never an edge */ \
X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS")
typedef enum {
internal/cbm/doclink_md.c
+23 -1
View File
@@ -493,6 +493,13 @@ static bool md_classify_path(char *s, size_t n, CBMDocLinkMdPath *out) {
if (n == 0) {
return false;
}
bool named = false;
for (size_t i = 0; i < n && !named; i++) {
named = md_word((unsigned char)s[i]);
}
if (!named) {
return false; /* `../`, `./`, `/`: a traversal or a root, naming no directory */
}
/* an import path (`github.com/x/y`, `k8s.io/client-go`): a host-like first
* segment -- dot-separated word groups -- and a last segment (as written,
* a trailing slash leaves it empty) without a file extension */
@@ -1090,13 +1097,28 @@ static void md_emit(md_scan_t *s, int syntax, const char *text, int n) {
cbm_doclinks_push(&s->ctx->result->doc_links, s->ctx->arena, link);
}
/* A code span's text: a reference when it names a path or a qualified name. */
/* A directory written as one name (`doc/`, `/docs`): no parent says whose it is. */
static bool md_one_dir(const CBMDocLinkMdPath *p) {
if (p->shape != CBM_DOCLINK_MD_DIR || !p->path) {
return false;
}
const char *s = p->path + (p->path[0] == '/');
const char *slash = strchr(s, '/');
return !slash || !slash[SKIP_ONE];
}
/* A code span's text: a reference when it names a path or a qualified name.
* A name alone -- a file (`config.yaml`) or a directory (`doc/`) -- is its own
* family: in the held-out audits it was the reader's own file or another
* project's directory as often as this repository's. */
static void md_emit_span(md_scan_t *s, const char *text, int n) {
CBMDocLinkMdPath p;
if (n > 0 &&
cbm_doclink_md_classify_span(text, (size_t)n, s->span_buf, MD_SPAN_MAX + SKIP_ONE, &p)) {
bool bare = p.shape == CBM_DOCLINK_MD_FILENAME || md_one_dir(&p);
md_emit(s,
p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_MD_CODE_NAME
: bare ? CBM_DOCLINK_MD_BARE_PATH
: CBM_DOCLINK_MD_CODE_PATH,
text, n);
}
src/pipeline/doc_links_md.c
+13 -9
View File
@@ -190,6 +190,17 @@ static int mdr_def_cmp(const void *a, const void *b) {
/* Every code definition of the graph, sorted by file. False when memory ran
* out. */
static bool mdr_build_defs(mdr_index_t *x, const cbm_gbuf_t *graph) {
/* the ADR records first: a repository of documents alone has records but
* no code definitions, and its supersedes links still need them */
const cbm_gbuf_node_t **adrs = NULL;
int nadr = 0;
if (cbm_gbuf_find_by_label(graph, "ADR", &adrs, &nadr) == 0) {
for (int i = 0; i < nadr; i++) {
if (adrs[i]->file_path && adrs[i]->file_path[0]) {
cbm_ht_set(x->adr_by_file, adrs[i]->file_path, (void *)adrs[i]);
}
}
}
int total = 0;
for (size_t l = 0; MDR_CODE_LABELS[l]; l++) {
const cbm_gbuf_node_t **nodes = NULL;
@@ -218,15 +229,6 @@ static bool mdr_build_defs(mdr_index_t *x, const cbm_gbuf_t *graph) {
}
}
qsort(x->defs, (size_t)x->ndefs, sizeof(*x->defs), mdr_def_cmp);
const cbm_gbuf_node_t **adrs = NULL;
int nadr = 0;
if (cbm_gbuf_find_by_label(graph, "ADR", &adrs, &nadr) == 0) {
for (int i = 0; i < nadr; i++) {
if (adrs[i]->file_path && adrs[i]->file_path[0]) {
cbm_ht_set(x->adr_by_file, adrs[i]->file_path, (void *)adrs[i]);
}
}
}
return true;
}
@@ -1142,6 +1144,7 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD
mdr_ref_t r = {0};
switch (link->syntax) {
case CBM_DOCLINK_MD_SUPERSEDES:
case CBM_DOCLINK_MD_SUPERSEDES_PROSE:
mdr_resolve_adr(x, graph, doc, link->raw, out);
return;
case CBM_DOCLINK_MD_LINK:
@@ -1169,6 +1172,7 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD
}
break;
}
case CBM_DOCLINK_MD_BARE_PATH:
case CBM_DOCLINK_MD_CODE_PATH: {
CBMDocLinkMdPath p;
if (!cbm_doclink_md_classify_span(link->raw, strlen(link->raw), buf, sizeof(buf), &p) ||
src/pipeline/doc_links_pdf.c
+7 -3
View File
@@ -327,9 +327,13 @@ static void qn_candidate(const pdfr_index_t *x, const cbm_gbuf_node_t *n, const
}
const char *rel = rel_qn(x, n->qualified_name);
char alt[PDFR_KEY_CAP];
bool via_rel = comp_suffix(rel, s);
bool via_alt = !via_rel && owner_name(x, n, alt, sizeof(alt)) && strcmp(alt, rel) != 0 &&
comp_suffix(alt, s);
bool owned = owner_name(x, n, alt, sizeof(alt));
/* A member whose QN leaves its owner out (a Go method's has no receiver)
* is named through the owner: `time.Now` is package time's function,
* never the method Now of a type that package time declares. */
bool ownerless_qn = owned && !comp_suffix(rel, alt);
bool via_rel = !ownerless_qn && comp_suffix(rel, s);
bool via_alt = !via_rel && owned && strcmp(alt, rel) != 0 && comp_suffix(alt, s);
if (via_rel || via_alt) {
ns_add(out, n);
if (via_rel && strcmp(rel, s) == 0) {
src/pipeline/doc_links_rst.c
+96 -22
View File
@@ -38,6 +38,7 @@
#include "doclink.h"
#include "foundation/arena.h"
#include "foundation/compat_fs.h" /* cbm_fopen */
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/mem_core.h"
@@ -49,11 +50,12 @@
#include <string.h>
enum {
RR_PATH_CAP = 1024, /* the longest path or dotted name looked up */
RR_RAW_CAP = 4096, /* the longest token record read */
RR_FIELDS = 8, /* fields of a token record */
RR_PIECES = 64, /* pieces of a dotted name */
RR_ALIAS_DEPTH = 12, /* re-exports followed (H3) */
RR_PATH_CAP = 1024, /* the longest path or dotted name looked up */
RR_INCLUDE_FILE_MAX = 8 * 1024 * 1024, /* an included file read for an open line range */
RR_RAW_CAP = 4096, /* the longest token record read */
RR_FIELDS = 8, /* fields of a token record */
RR_PIECES = 64, /* pieces of a dotted name */
RR_ALIAS_DEPTH = 12, /* re-exports followed (H3) */
RR_TABLE_INIT = 1024,
};
@@ -89,7 +91,8 @@ typedef struct {
typedef struct {
const char *project;
size_t project_len;
void *md; /* the Markdown resolver's index: code spans, segments, folders */
const char *repo; /* the repository root: an open line range reads its file */
void *md; /* the Markdown resolver's index: code spans, segments, folders */
const char **run_paths;
int run_count;
CBMArena arena;
@@ -345,6 +348,7 @@ static void *rr_build(const cbm_doclink_build_in_t *in) {
cbm_arena_init(&x->arena);
x->project = in->ctx ? in->ctx->project_name : NULL;
x->project_len = x->project ? strlen(x->project) : 0;
x->repo = in->ctx ? in->ctx->repo_path : NULL;
x->run_count = in->run_file_count;
x->run_paths = in->run_file_count > 0
? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER,
@@ -1366,6 +1370,65 @@ static bool rr_doc_path(const rr_index_t *x, const char *doc, const char *arg, c
return rr_norm(dir, arg, out, cap);
}
/* An included file's text, read from the indexed file on disk (at most
* RR_INCLUDE_FILE_MAX bytes). NULL when it cannot be read; the caller frees
* it (cbm_free, OTHER). */
static char *rr_read(const rr_index_t *x, const char *rel, size_t *n) {
*n = 0;
if (!x->repo) {
return NULL;
}
char abs[RR_PATH_CAP * PAIR_LEN];
snprintf(abs, sizeof(abs), "%s/%s", x->repo, rel);
FILE *f = cbm_fopen(abs, "rb");
if (!f) {
return NULL;
}
char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, RR_INCLUDE_FILE_MAX);
*n = buf ? fread(buf, SKIP_ONE, RR_INCLUDE_FILE_MAX, f) : 0;
(void)fclose(f);
return buf;
}
static uint32_t rr_number(const char **pp) {
uint32_t v = 0;
const char *p = *pp;
while (*p >= '0' && *p <= '9') {
v = v < UINT32_MAX / 10 ? v * 10 + (uint32_t)(*p - '0') : UINT32_MAX;
p++;
}
*pp = p;
return v;
}
/* `:lines: 1,3,5-10,20-`: the span of all its ranges. `N-` runs to the file's
* end (*b = UINT32_MAX) and `-N` from its start. 0 and 0 when it holds none. */
static void rr_lines(const char *v, uint32_t *a, uint32_t *b) {
*a = 0;
*b = 0;
for (const char *p = v; *p;) {
if (*p == ',' || *p == ' ') {
p++;
continue;
}
bool has_start = *p >= '0' && *p <= '9';
uint32_t s = has_start ? rr_number(&p) : 0;
uint32_t e = s;
if (*p == '-') {
p++;
e = *p >= '0' && *p <= '9' ? rr_number(&p) : UINT32_MAX;
s = has_start ? s : SKIP_ONE;
} else if (!has_start) {
p++; /* not a range: skip the character */
continue;
}
if (s > 0 && e >= s) {
*a = (*a == 0 || s < *a) ? s : *a;
*b = e > *b ? e : *b;
}
}
}
static void rr_literalinclude(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc,
const rr_rec_t *r, cbm_doclink_outcome_t *out) {
const char *arg = r->f[1];
@@ -1410,22 +1473,25 @@ static void rr_literalinclude(const rr_index_t *x, const cbm_gbuf_t *graph, cons
}
uint32_t a = 0;
uint32_t b = 0;
for (const char *p = lines; *p;) {
if (*p >= '0' && *p <= '9') {
uint32_t v = 0;
while (*p >= '0' && *p <= '9') {
v = v < UINT32_MAX / 10 ? v * 10 + (uint32_t)(*p - '0') : UINT32_MAX;
p++;
}
a = (a == 0 || v < a) ? v : a;
b = v > b ? v : b;
} else {
p++;
}
}
rr_lines(lines, &a, &b);
if (a > 0) {
const cbm_gbuf_node_t *seg = cbm_doclink_md_segment(x->md, rel, a, b, NULL);
rr_edge(out, seg ? seg : file, true, a, b);
if (b == UINT32_MAX) {
/* `N-`: to the file's end, which only its text knows */
size_t tn = 0;
char *text = rr_read(x, rel, &tn);
if (text) {
uint32_t nl = 0;
for (size_t i = 0; i < tn; i++) {
nl += text[i] == '\n';
}
nl += tn > 0 && text[tn - SKIP_ONE] != '\n';
b = nl >= a ? nl : a;
cbm_free(CBM_MEM_CLASS_OTHER, text);
}
}
const cbm_gbuf_node_t *seg =
b == UINT32_MAX ? NULL : cbm_doclink_md_segment(x->md, rel, a, b, NULL);
rr_edge(out, seg ? seg : file, true, a, b == UINT32_MAX ? 0 : b);
return;
}
rr_edge(out, file, true, 0, 0);
@@ -1525,6 +1591,13 @@ static void rr_py_object(const rr_index_t *x, const cbm_gbuf_t *graph, const cha
char full[RR_PATH_CAP];
snprintf(full, sizeof(full), "%s%s%s", m, m[0] ? "." : "", fullname);
rr_res_t res = rr_dotted(x, graph, full);
/* `.. py:function:: autofit` declares a definition inside a module: a
* module, folder or file of that name is another entity of the name */
if (res.kind == RR_HIT && res.node &&
(strcmp(res.node->label, "Module") == 0 || strcmp(res.node->label, "Folder") == 0 ||
strcmp(res.node->label, "File") == 0)) {
res = (rr_res_t){RR_NONE, NULL, false};
}
if (res.kind != RR_HIT) {
res = rr_py_resolve(x, graph, fullname, dname, m, NULL, false, true);
}
@@ -1601,7 +1674,8 @@ static void rr_resolve(const void *index, void *state, int run_file, const CBMDo
rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE);
return;
}
if (link->syntax == CBM_DOCLINK_MD_SUPERSEDES) {
if (link->syntax == CBM_DOCLINK_MD_SUPERSEDES ||
link->syntax == CBM_DOCLINK_MD_SUPERSEDES_PROSE) {
/* a reST ADR's own supersedes statement (doc_adr.c) */
cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, link, graph, out);
return;
tests/test_doc_links_md.c
+152 -9
View File
@@ -125,8 +125,14 @@ TEST(doc_links_md_extract_tokens) {
t = md_token(r, "app/applications.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_PATH);
ASSERT_NOT_NULL(md_token(r, "app/middleware/"));
ASSERT_NOT_NULL(md_token(r, "/docs")); /* a token; the resolver calls it a route */
t = md_token(r, "app/middleware/");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH);
/* a token (the resolver calls it a route); one directory name alone is a
* bare path: in a held-out audit `doc/` was another project's as often as not */
t = md_token(r, "/docs");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH);
/* an image, a URL: no tokens */
ASSERT_NULL(md_token(r, "img/a.png"));
ASSERT_NULL(md_token(r, "https://example.org/a/b.py"));
@@ -140,7 +146,11 @@ TEST(doc_links_md_extract_tokens) {
ASSERT_NOT_NULL(md_token(r, "pkg.Config.Name"));
ASSERT_NULL(md_token(r, "self.Name")); /* one identifier after `self.`: a bare name */
ASSERT_NOT_NULL(md_token(r, "os.path.join"));
ASSERT_NOT_NULL(md_token(r, "main.py"));
/* a file name alone is its own family: in the held-out audit every wrong
* code path was one (`config.yaml`: the reader's own file) */
t = md_token(r, "main.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH);
/* fenced code and HTML comments are not scanned; HTML <code> is */
ASSERT_NULL(md_token(r, "app/fenced.py"));
ASSERT_NULL(md_token(r, "app/commented.py"));
@@ -231,6 +241,11 @@ TEST(doc_links_md_classify_span) {
{"\"a/b.py\"", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"pkg/Type.Member", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"@scope/pkg", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
/* a traversal or a root names no directory (a held-out link: `../` in a
* warning about attacker-controlled paths) */
{"../", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"./", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"../..", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
const md_case_t *c = &cases[i];
@@ -348,8 +363,9 @@ TEST(doc_links_md_resolve) {
ASSERT_STR_EQ(syntax, "code_name");
dm_row(db, "docs/guide.md", "test_routing.only_in_tests", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "test_only_target");
dm_row(db, "docs/guide.md", "main.py", reason, sizeof(reason), NULL, 0);
dm_row(db, "docs/guide.md", "main.py", reason, sizeof(reason), syntax, sizeof(syntax));
ASSERT_STR_EQ(reason, "ambiguous"); /* lib/main.py, but not next to the document */
ASSERT_STR_EQ(syntax, "bare_path");
dm_row(db, "docs/guide.md", "../app/gone.py", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "missing");
dm_row(db, "docs/guide.md", "app/html.py", reason, sizeof(reason), NULL, 0);
@@ -582,21 +598,146 @@ TEST(doc_links_md_ship_gate) {
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_LINK));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_ROLE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_ADOC_INCLUDE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_LITERALINCLUDE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_CODE_PATH));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_CODE_PATH));
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdgate_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
md_write_fixture(tmp);
/* a file name alone, next to its file: it resolves, and its tier holds it */
th_write_file(TH_PATH(tmp, "pkg/README.md"), "# Pkg\n\nSee `config.go`.\n");
char db[512];
snprintf(db, sizeof(db), "%s/md.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
char props[512];
ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 0);
ASSERT_TRUE(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'code_path'") > 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND "
"properties LIKE '%\"syntax\":\"code_path\"%'"),
ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'code_path'"),
0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'bare_path' "
"AND rel_path = 'pkg/README.md'"),
1);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND "
"properties LIKE '%\"syntax\":\"bare_path\"%'"),
0);
th_cleanup(tmp);
PASS();
}
/* A held-out finding: a spec template's combined field `Supersedes / Depends
* on: [Spec 2](...)` states no supersession (the record depended on the one it
* named); the plain statement next to it still does. */
TEST(doc_links_md_adr_label_choice) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrl_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
"# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
"# Next\n\n## Status\n\nAccepted\n\n"
"Supersedes / Depends on: [ADR-0001](0001-base.md)\n\n"
"## Decision\n\nWe add B.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"), "# Last\n\n## Status\n\nAccepted\n\n"
"Supersedes [ADR-0001](0001-base.md).\n\n"
"## Decision\n\nWe use C.\n");
char db[512];
snprintf(db, sizeof(db), "%s/adr.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0002-next.md'"),
0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0003-last.md'"),
1);
th_cleanup(tmp);
PASS();
}
static int md_supersedes_from(const char *db, const char *file, const char *target) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' "
"AND s.file_path = 'docs/adr/%s' AND t.file_path LIKE 'docs/adr/%s'",
file, target);
return dm_count(db, sql);
}
static int md_supersedes_syntax(const char *db, const char *file, const char *syntax) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/%s' "
"AND e.properties LIKE '%%\"syntax\":\"%s\"%%'",
file, syntax);
return dm_count(db, sql);
}
/* Held-out findings: a field whose value is "none" goes on to name records it
* extends; a record reports another record's supersession (`ADR 0002
* supersedes ...`, a table cell `0002 (supersedes ...)`); the record after a
* statement's full stop is no target. The record's own number as the subject
* is its own statement. */
TEST(doc_links_md_adr_reported) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrr_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
"# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
"# Next\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use B.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-none.md"),
"# None\n\n- Status: Accepted\n"
"- Supersedes: none (it extends [ADR 0001](0001-base.md))\n\n"
"## Decision\n\nWe add C.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0004-na.md"),
"# NA\n\n- Status: Accepted\n- Supersedes: **N/A** -- refines ADR-0002\n\n"
"## Decision\n\nWe add D.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0005-report.md"),
"# Report\n\n## Status\n\nAccepted\n\n## Context\n\n"
"ADR 0002 supersedes ADR 0001's first decision.\n\n"
"| Row | Owner |\n|---|---|\n| retries | 0002 (supersedes ADR 0001's rule) |\n\n"
"[ADR 0002](0002-next.md) supersedes ADR 0001 too.\n\n"
"## Decision\n\nWe audit.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0006-own.md"),
"# Own\n\n## Status\n\nAccepted\n\n## Decision\n\n"
"ADR 0006 supersedes ADR 0001.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0007-stop.md"),
"# Stop\n\n## Status\n\nAccepted\n\n## Decision\n\n"
"> This record supersedes ADR-0001. ADR 0002 is an orthogonal decision.\n");
/* adr-tools' own form: the link text holds the record's number and title */
th_write_file(TH_PATH(tmp, "docs/adr/0008-tools.md"),
"# 8. Tools\n\n## Status\n\nAccepted\n\n"
"Supersedes [2. Next](0002-next.md)\n\n## Decision\n\nWe use E.\n");
/* statements in a field table and a bold field */
th_write_file(TH_PATH(tmp, "docs/adr/0009-table.md"),
"# Table\n\n| Field | Value |\n|---|---|\n| Status | Accepted |\n"
"| Supersedes | [ADR 0001](0001-base.md) |\n\n## Decision\n\nWe use F.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0010-field.md"),
"# Field\n\n- Status: Accepted\n- **Supersedes:** ADR-0002\n\n"
"## Decision\n\nWe use G.\n");
char db[512];
snprintf(db, sizeof(db), "%s/adr.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
ASSERT_EQ(md_supersedes_from(db, "0003-none.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0004-na.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0005-report.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0006-own.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0002-next.md"), 0);
ASSERT_EQ(md_supersedes_from(db, "0008-tools.md", "0002-next.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0009-table.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0010-field.md", "0002-next.md"), 1);
/* a statement opens its line or field and names a record directly; the
* same words in running prose are the held family supersedes_prose */
ASSERT_EQ(md_supersedes_syntax(db, "0006-own.md", "supersedes_prose"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0007-stop.md", "supersedes_prose"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0008-tools.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0009-table.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0010-field.md", "supersedes"), 1);
th_cleanup(tmp);
PASS();
}
@@ -609,6 +750,8 @@ SUITE(doc_links_md) {
RUN_TEST(doc_links_md_resolve);
RUN_TEST(doc_links_md_incremental);
RUN_TEST(doc_links_md_adr);
RUN_TEST(doc_links_md_adr_label_choice);
RUN_TEST(doc_links_md_adr_reported);
cbm_doclink_test_reset_ships();
RUN_TEST(doc_links_md_ship_gate);
}
tests/test_doc_links_pdf.c
+62 -18
View File
@@ -1085,29 +1085,35 @@ TEST(pdf_links_pipeline) {
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_path\""));
ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\""));
ASSERT_NOT_NULL(strstr(props, "\"line\":1"));
/* qualified names (pdf_qn) did not pass the held-out audit (77 of 80 correct,
* Wilson 95 % low 0.896 < 0.90): no edge, a below_bar_tier row each */
const char *const qns[] = {"server.Handler.Serve", "app.models.User.save", "server.NewHandler",
"app.models"};
for (size_t i = 0; i < sizeof(qns) / sizeof(qns[0]); i++) {
char qreason[64];
char qsyntax[32];
dm_row(db, "docs/design.pdf", qns[i], qreason, sizeof(qreason), qsyntax, sizeof(qsyntax));
ASSERT_STR_EQ(qreason, "below_bar_tier");
ASSERT_STR_EQ(qsyntax, "pdf_qn");
}
/* qualified names (pdf_qn) passed a second held-out audit (97 of 99
* correct, Wilson 95 % low 0.929) once a member whose QN leaves its owner
* out binds only through the owner. A Go method through its receiver: */
dm_edge(db, "docs.design.page_1", "pkg.server.Serve", props, sizeof(props), &n);
ASSERT_EQ(n, 0);
ASSERT_EQ(n, 1);
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\""));
ASSERT_NOT_NULL(strstr(props, "\"tier\":\"unique\""));
/* qn-exact */
dm_edge(db, "docs.design.page_1", "app.models.User.save", props, sizeof(props), &n);
ASSERT_EQ(n, 0);
/* page 2: a file name */
ASSERT_EQ(n, 1);
ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\""));
ASSERT_NOT_NULL(strstr(props, "\"line\":2"));
/* page 2: a Go function, a file name */
dm_edge(db, "docs.design.page_2", "pkg.server.NewHandler", props, sizeof(props), &n);
ASSERT_EQ(n, 1);
dm_edge(db, "docs.design.page_2", "app.models.py.__file__", props, sizeof(props), &n);
ASSERT_EQ(n, 1);
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_file\""));
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id WHERE "
"e.type='MENTIONS' AND s.file_path='docs/design.pdf' AND "
"e.properties LIKE '%\"syntax\":\"pdf_qn\"%'"),
0);
/* a module by its qualified name (a Module node is named by its path) */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN "
"nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND "
"t.label='Module' AND t.file_path='app/models.py' AND s.name='page 2' "
"AND e.properties LIKE '%\"tier\":\"exact\"%'"),
1);
/* back navigation: the method knows the page that names it */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN "
"nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND t.name='Serve' "
"AND s.name='page 1'"),
1);
/* hygiene: a test target is a row, a data file nothing; a fixture PDF links nothing */
char reason[64];
dm_row(db, "docs/design.pdf", "tests/test_api.py", reason, sizeof(reason), NULL, 0);
@@ -1136,6 +1142,43 @@ TEST(pdf_links_pipeline) {
PASS();
}
/* A held-out finding of the PDF audit: a Go method's QN has no receiver, so
* `time.Now` (the standard library's function, in a spec) matched the method
* Now of a type that the repository's own package time declares. A method is
* named through its owner: `time.Provider.Now` still binds it. */
TEST(pdf_qn_method_through_owner) {
cbm_doclink_test_set_ships(CBM_DOCLINK_PDF_QN, true);
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlpdfq_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
th_write_file(TH_PATH(repo, "pkg/time/provider.go"),
"package time\n"
"\n"
"type Provider struct{}\n"
"\n"
"func (p *Provider) Now() int { return 0 }\n");
char path[1024];
snprintf(path, sizeof(path), "%s/docs/spec.pdf", repo);
const char *pages[2] = {"Go's clock: time.Now is the standard library's.",
"Our clock: time.Provider.Now wraps it."};
tp_write_pdf(path, pages, 2);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
char props[512];
int n;
dm_edge(db, "docs.spec.page_1", "pkg.time.Now", props, sizeof(props), &n);
ASSERT_EQ(n, 0);
dm_edge(db, "docs.spec.page_2", "pkg.time.Now", props, sizeof(props), &n);
ASSERT_EQ(n, 1);
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"pdf_qn\""));
th_cleanup(tmp);
cbm_doclink_test_reset_ships();
PASS();
}
/* A page's docstring keeps 256 KiB of its text and says where it was cut; a
* page's mentions still come from the whole text. */
TEST(pdf_page_doc_bounded) {
@@ -1267,6 +1310,7 @@ SUITE(doc_links_pdf) {
RUN_TEST(pdf_scan_mentions);
RUN_TEST(pdf_scan_join_linear);
RUN_TEST(pdf_links_pipeline);
RUN_TEST(pdf_qn_method_through_owner);
RUN_TEST(pdf_page_doc_bounded);
RUN_TEST(pdf_snippet_is_page_text);
RUN_TEST(pdf_links_incremental);
tests/test_doc_links_rst.c
+45
View File
@@ -445,11 +445,56 @@ TEST(rst_links_incremental) {
PASS();
}
/* Two held-out findings (supplementary audit): `.. py:function:: autofit` on a
* page without a module bound the example script module examples/autofit.py
* (an object directive declares a definition, never a module), and
* `:lines: 5-` (line 5 to the file's end) was read as line 5 alone. */
TEST(rst_object_and_open_lines) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrsto_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
th_write_file(TH_PATH(repo, "examples/autofit.py"), "print('autofit example')\n");
th_write_file(TH_PATH(repo, "examples/tail.py"), "import os\n" /* 1 */
"\n" /* 2 */
"\n" /* 3 */
"class Tail:\n" /* 4 */
" def a(self):\n" /* 5 */
" return 1\n" /* 6 */
"\n" /* 7 */
" def b(self):\n" /* 8 */
" return 2\n"); /* 9 */
th_write_file(TH_PATH(repo, "docs/conf.py"), "project = 'x'\n");
th_write_file(TH_PATH(repo, "docs/worksheet.rst"), "Worksheet\n"
"=========\n"
"\n"
".. py:function:: autofit()\n"
"\n"
" Fit the columns.\n"
"\n"
".. literalinclude:: ../examples/tail.py\n"
" :lines: 5-\n");
ASSERT_EQ(dm_index(repo, db, NULL), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE "
"e.type='MENTIONS' AND t.file_path='examples/autofit.py'"),
0);
ASSERT_EQ(rst_rows(db, ".. py:function:: autofit", "missing"), 1);
/* lines 5 to 9 lie in class Tail; line 5 alone lies in method a */
ASSERT_TRUE(
rst_edge(db, "docs.worksheet.Worksheet", "examples.tail.Tail", "\"target_lines\":[5,9]"));
th_cleanup(tmp);
PASS();
}
SUITE(doc_links_rst) {
dm_ship_held_families();
RUN_TEST(rst_scan_structure);
RUN_TEST(rst_python_scope_blob);
RUN_TEST(rst_links_pipeline);
RUN_TEST(rst_object_and_open_lines);
RUN_TEST(rst_adr_record);
RUN_TEST(rst_links_incremental);
cbm_doclink_test_reset_ships();
tests/test_doc_mentions_helpers.h
+3 -4
View File
@@ -416,10 +416,9 @@ static inline int dm_reason_lines(const char *db, const char *block) {
* so it is the resolution an audit would ship; doc_links_md_ship_gate tests
* the production gate itself. */
static const int DM_HELD_FAMILIES[] = {
CBM_DOCLINK_MD_CODE_PATH, CBM_DOCLINK_MD_SUPERSEDES, CBM_DOCLINK_RST_OBJECT,
CBM_DOCLINK_RST_INCLUDE, CBM_DOCLINK_RST_LITERALINCLUDE, CBM_DOCLINK_RST_KERNEL_DOC,
CBM_DOCLINK_RST_CODE_PATH, CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH,
CBM_DOCLINK_ADOC_CODE_NAME,
CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES, CBM_DOCLINK_MD_SUPERSEDES_PROSE,
CBM_DOCLINK_RST_OBJECT, CBM_DOCLINK_RST_INCLUDE, CBM_DOCLINK_RST_KERNEL_DOC,
CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH, CBM_DOCLINK_ADOC_CODE_NAME,
};
static inline void dm_ship_held_families(void) {