mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
fix(pipeline): address doclinks review feedback
Replace raw malloc/free with cbm_alloc/cbm_free (CBM_MEM_CLASS_EXTRACT) and a growable CBM_DYN_ARRAY for matches. Remove the 4KiB line-copy cap; lines are split in place instead of copied. Remove the DOCLINK_MAX_REFS(256) cap; the ref vector now grows, is sorted by target id, then collapsed. Track reference kind (bare, relative, rooted) so /x never attempts a directory join and ./x never falls back to root. Report per-file skips via cbm_pipeline_add_file_error and cbm_log_warn, and check cbm_pipeline_check_cancel in the per-file loop. Add negative-control tests for all of the above, plus CRLF handling, fenced-code-block pinning, bare URLs, backtick-basename exactness, and ".." escaping. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Signed-off-by: Clément Deust <clement.deust@gmail.com>
This commit is contained in:
1 parent
f3bdf11a03
commit
6ae6cd4c3f
5 files changed
+432
-80
No files matched your search
+154
-77
@@ -30,6 +30,8 @@
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/mem_core.h"
|
||||
#include "foundation/dyn_array.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
@@ -50,8 +52,7 @@
|
||||
#define DOCLINK_EDGE_TYPE "REFERENCES_FILE"
|
||||
|
||||
enum {
|
||||
DOCLINK_MAX_REFS = CBM_SZ_256, /* distinct targets per referencing file */
|
||||
DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */
|
||||
DOCLINK_MAX_SEGS = CBM_SZ_64, /* path segments during normalization */
|
||||
};
|
||||
|
||||
/* ── Path classification ─────────────────────────────────────────── */
|
||||
@@ -71,23 +72,51 @@ static bool doclink_is_markdown_path(const char *path) {
|
||||
return ext && (strcmp(ext, ".md") == 0 || strcmp(ext, ".mdx") == 0);
|
||||
}
|
||||
|
||||
/* ── File reading (mirrors pass_semantic.c read_file, minus TS pad) ── */
|
||||
|
||||
static char *doclink_read_file(const char *path) {
|
||||
/* ── File reading (mirrors pass_definitions.c read_file, minus TS pad) ──
|
||||
*
|
||||
* Reports *out_status so the caller can attribute a skip to the right
|
||||
* reason (open failure vs. oversized vs. OOM) instead of a silent drop —
|
||||
* hitting a limit here degrades to a REPORTED skip, per limits.h. */
|
||||
static char *doclink_read_file(const char *path, long *out_size, cbm_read_status_t *out_status) {
|
||||
if (out_size) {
|
||||
*out_size = 0;
|
||||
}
|
||||
if (out_status) {
|
||||
*out_status = CBM_READ_OK;
|
||||
}
|
||||
FILE *f = cbm_fopen(path, "rb");
|
||||
if (!f) {
|
||||
if (out_status) {
|
||||
*out_status = CBM_READ_OPEN_FAIL;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
(void)fseek(f, 0, SEEK_END);
|
||||
long size = ftell(f);
|
||||
(void)fseek(f, 0, SEEK_SET);
|
||||
if (size <= 0 || size > cbm_max_file_bytes()) {
|
||||
if (out_size) {
|
||||
*out_size = size;
|
||||
}
|
||||
if (size <= 0) {
|
||||
(void)fclose(f);
|
||||
if (out_status) {
|
||||
*out_status = CBM_READ_EMPTY;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
char *buf = malloc((size_t)size + SKIP_ONE);
|
||||
if (size > cbm_max_file_bytes()) {
|
||||
(void)fclose(f);
|
||||
if (out_status) {
|
||||
*out_status = CBM_READ_OVERSIZED;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
char *buf = cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE);
|
||||
if (!buf) {
|
||||
(void)fclose(f);
|
||||
if (out_status) {
|
||||
*out_status = CBM_READ_OOM;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
size_t nread = fread(buf, SKIP_ONE, (size_t)size, f);
|
||||
@@ -140,33 +169,45 @@ static bool doclink_normalize(const char *in, char *out, size_t out_sz) {
|
||||
return out_len > 0;
|
||||
}
|
||||
|
||||
/* Per-file accumulator: dedupes repeated references to the same target. */
|
||||
/* One raw match before dedup. Collected in a growable vector (no per-file
|
||||
* cap) and collapsed by doclink_flush(). */
|
||||
typedef struct {
|
||||
int64_t target_id;
|
||||
int count;
|
||||
double confidence;
|
||||
const char *strategy; /* static string literal */
|
||||
} doclink_ref_t;
|
||||
} doclink_match_t;
|
||||
|
||||
typedef struct {
|
||||
cbm_gbuf_t *gb;
|
||||
CBMHashTable *files_by_path; /* rel_path → cbm_gbuf_node_t* (borrowed) */
|
||||
const cbm_gbuf_node_t *src; /* referencing File node */
|
||||
char src_dir[CBM_SZ_512]; /* its directory ("" at repo root) */
|
||||
doclink_ref_t refs[DOCLINK_MAX_REFS];
|
||||
int ref_count;
|
||||
bool truncated;
|
||||
CBM_DYN_ARRAY(doclink_match_t) matches;
|
||||
} doclink_ctx_t;
|
||||
|
||||
/* Resolve a reference against the referencing file's directory, then the
|
||||
* repo root, mirroring how humans write doc links. Returns the already-
|
||||
* indexed File node or NULL — unresolvable references are dropped. */
|
||||
/* How a reference was written, which decides where it may resolve. Mixing
|
||||
* these up let a rooted "/x" still match a same-named file in the
|
||||
* referencing directory, and let an explicit "./x" fall back to an
|
||||
* unrelated same-named file at the repo root. */
|
||||
typedef enum {
|
||||
DOCLINK_REF_BARE = 0, /* "x": referencing dir, then repo root */
|
||||
DOCLINK_REF_RELATIVE, /* "./x": referencing dir only */
|
||||
DOCLINK_REF_ROOTED, /* "/x": repo root only */
|
||||
} doclink_ref_kind_t;
|
||||
|
||||
/* Resolve a reference against the referencing file's directory and/or the
|
||||
* repo root, per its kind. Returns the already-indexed File node or NULL —
|
||||
* unresolvable references are dropped. */
|
||||
static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref) {
|
||||
const char *r = ref;
|
||||
while (r[0] == '.' && r[SKIP_ONE] == '/') {
|
||||
r += PAIR_LEN;
|
||||
}
|
||||
if (r[0] == '/') {
|
||||
doclink_ref_kind_t kind = DOCLINK_REF_BARE;
|
||||
if (r[0] == '.' && r[SKIP_ONE] == '/') {
|
||||
kind = DOCLINK_REF_RELATIVE;
|
||||
while (r[0] == '.' && r[SKIP_ONE] == '/') {
|
||||
r += PAIR_LEN;
|
||||
}
|
||||
} else if (r[0] == '/') {
|
||||
kind = DOCLINK_REF_ROOTED;
|
||||
r++; /* "/docs/x.md" is repo-root-relative by doc convention */
|
||||
}
|
||||
if (r[0] == '\0') {
|
||||
@@ -174,7 +215,7 @@ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref
|
||||
}
|
||||
|
||||
char norm[CBM_SZ_512];
|
||||
if (dc->src_dir[0] != '\0') {
|
||||
if (kind != DOCLINK_REF_ROOTED && dc->src_dir[0] != '\0') {
|
||||
char joined[CBM_SZ_512];
|
||||
int n = snprintf(joined, sizeof(joined), "%s/%s", dc->src_dir, r);
|
||||
if (n > 0 && (size_t)n < sizeof(joined) && doclink_normalize(joined, norm, sizeof(norm))) {
|
||||
@@ -184,61 +225,67 @@ static const cbm_gbuf_node_t *doclink_resolve(doclink_ctx_t *dc, const char *ref
|
||||
}
|
||||
}
|
||||
}
|
||||
if (doclink_normalize(r, norm, sizeof(norm))) {
|
||||
if (kind != DOCLINK_REF_RELATIVE && doclink_normalize(r, norm, sizeof(norm))) {
|
||||
return cbm_ht_get(dc->files_by_path, norm);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Record one match. Same-pair repeats bump the count; a higher-confidence
|
||||
* strategy upgrades the edge's confidence + strategy label. */
|
||||
/* Record one match. No cap, no dedup here — dc->matches is a plain append
|
||||
* log; doclink_flush() sorts and collapses same-target matches so a
|
||||
* generated file with hundreds of distinct targets loses nothing. */
|
||||
static void doclink_record(doclink_ctx_t *dc, const cbm_gbuf_node_t *target, double confidence,
|
||||
const char *strategy) {
|
||||
if (!target || target->id == dc->src->id) {
|
||||
return; /* never self-reference */
|
||||
}
|
||||
for (int i = 0; i < dc->ref_count; i++) {
|
||||
if (dc->refs[i].target_id == target->id) {
|
||||
dc->refs[i].count++;
|
||||
if (confidence > dc->refs[i].confidence) {
|
||||
dc->refs[i].confidence = confidence;
|
||||
dc->refs[i].strategy = strategy;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (dc->ref_count >= DOCLINK_MAX_REFS) {
|
||||
dc->truncated = true;
|
||||
return;
|
||||
}
|
||||
dc->refs[dc->ref_count].target_id = target->id;
|
||||
dc->refs[dc->ref_count].count = SKIP_ONE;
|
||||
dc->refs[dc->ref_count].confidence = confidence;
|
||||
dc->refs[dc->ref_count].strategy = strategy;
|
||||
dc->ref_count++;
|
||||
doclink_match_t m = {.target_id = target->id, .confidence = confidence, .strategy = strategy};
|
||||
cbm_da_push(&dc->matches, m);
|
||||
}
|
||||
|
||||
/* Emit accumulated references as REFERENCES_FILE edges. Returns edge count. */
|
||||
static int doclink_match_cmp(const void *a, const void *b) {
|
||||
int64_t ta = ((const doclink_match_t *)a)->target_id;
|
||||
int64_t tb = ((const doclink_match_t *)b)->target_id;
|
||||
return (ta > tb) - (ta < tb);
|
||||
}
|
||||
|
||||
/* Emit accumulated references as REFERENCES_FILE edges. Sorts by target id
|
||||
* (O(M log M)) and collapses same-target runs: count = occurrences, the
|
||||
* highest-confidence match in the run wins strategy + confidence. Returns
|
||||
* edge count. */
|
||||
static int doclink_flush(doclink_ctx_t *dc) {
|
||||
if (dc->matches.count > 1) {
|
||||
qsort(dc->matches.items, (size_t)dc->matches.count, sizeof(dc->matches.items[0]),
|
||||
doclink_match_cmp);
|
||||
}
|
||||
|
||||
int emitted = 0;
|
||||
for (int i = 0; i < dc->ref_count; i++) {
|
||||
int i = 0;
|
||||
while (i < dc->matches.count) {
|
||||
int64_t target_id = dc->matches.items[i].target_id;
|
||||
double confidence = dc->matches.items[i].confidence;
|
||||
const char *strategy = dc->matches.items[i].strategy;
|
||||
int count = SKIP_ONE;
|
||||
int j = i + SKIP_ONE;
|
||||
while (j < dc->matches.count && dc->matches.items[j].target_id == target_id) {
|
||||
count++;
|
||||
if (dc->matches.items[j].confidence > confidence) {
|
||||
confidence = dc->matches.items[j].confidence;
|
||||
strategy = dc->matches.items[j].strategy;
|
||||
}
|
||||
j++;
|
||||
}
|
||||
|
||||
char props[CBM_SZ_256];
|
||||
(void)snprintf(props, sizeof(props),
|
||||
"{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}",
|
||||
dc->refs[i].strategy, dc->refs[i].confidence, dc->refs[i].count);
|
||||
if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, dc->refs[i].target_id, DOCLINK_EDGE_TYPE,
|
||||
props) > 0) {
|
||||
"{\"strategy\":\"%s\",\"confidence\":%.2f,\"count\":%d}", strategy,
|
||||
confidence, count);
|
||||
if (cbm_gbuf_insert_edge(dc->gb, dc->src->id, target_id, DOCLINK_EDGE_TYPE, props) > 0) {
|
||||
emitted++;
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
if (dc->truncated) {
|
||||
char cap_buf[CBM_SZ_16];
|
||||
(void)snprintf(cap_buf, sizeof(cap_buf), "%d", DOCLINK_MAX_REFS);
|
||||
cbm_log_info("doclinks.truncated", "file", dc->src->file_path ? dc->src->file_path : "",
|
||||
"cap", cap_buf);
|
||||
}
|
||||
dc->ref_count = 0;
|
||||
dc->truncated = false;
|
||||
cbm_da_clear(&dc->matches);
|
||||
return emitted;
|
||||
}
|
||||
|
||||
@@ -380,11 +427,15 @@ static void doclink_scan_md_line(doclink_ctx_t *dc, char *line) {
|
||||
|
||||
/* ── Per-file driver ─────────────────────────────────────────────── */
|
||||
|
||||
/* Scan one referencing file's content line by line and emit its edges. */
|
||||
static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, const char *source) {
|
||||
/* Scan one referencing file's content line by line and emit its edges.
|
||||
* `source` is the caller's private mutable buffer (freshly read for this
|
||||
* file, never reused): lines are terminated in place ('\n' -> '\0') and
|
||||
* scanned by pointer, so there is no fixed-size copy and no line-length
|
||||
* cap to straddle a token across (a prior CBM_SZ_4K copy buffer could cut
|
||||
* "src/foo.cpp" to "src/foo.c" at the boundary and bind to the wrong
|
||||
* file). A trailing '\r' (CRLF) is trimmed the same way. */
|
||||
static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, char *source) {
|
||||
dc->src = node;
|
||||
dc->ref_count = 0;
|
||||
dc->truncated = false;
|
||||
dc->src_dir[0] = '\0';
|
||||
const char *slash = strrchr(node->file_path, '/');
|
||||
if (slash) {
|
||||
@@ -396,18 +447,20 @@ static int doclink_scan_file(doclink_ctx_t *dc, const cbm_gbuf_node_t *node, con
|
||||
dc->src_dir[dlen] = '\0';
|
||||
}
|
||||
|
||||
const char *p = source;
|
||||
char line[CBM_SZ_4K];
|
||||
char *p = source;
|
||||
while (*p) {
|
||||
const char *eol = strchr(p, '\n');
|
||||
size_t line_len = eol ? (size_t)(eol - p) : strlen(p);
|
||||
if (line_len >= sizeof(line)) {
|
||||
line_len = sizeof(line) - SKIP_ONE;
|
||||
char *line = p;
|
||||
char *eol = strchr(p, '\n');
|
||||
if (eol) {
|
||||
*eol = '\0';
|
||||
p = eol + SKIP_ONE;
|
||||
} else {
|
||||
p += strlen(p);
|
||||
}
|
||||
size_t line_len = strlen(line);
|
||||
if (line_len > 0 && line[line_len - SKIP_ONE] == '\r') {
|
||||
line[line_len - SKIP_ONE] = '\0';
|
||||
}
|
||||
memcpy(line, p, line_len);
|
||||
line[line_len] = '\0';
|
||||
p = eol ? eol + SKIP_ONE : p + line_len;
|
||||
|
||||
doclink_scan_md_line(dc, line);
|
||||
}
|
||||
return doclink_flush(dc);
|
||||
@@ -426,10 +479,16 @@ static bool doclink_has_doc_files(const cbm_gbuf_node_t *const *files, int file_
|
||||
}
|
||||
|
||||
/* Scan every markdown File node's on-disk content, emitting edges.
|
||||
* md_edges receives the emitted edge count. */
|
||||
static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path,
|
||||
* md_edges receives the emitted edge count. Checks cancellation once per
|
||||
* file — on a docs-heavy repo this is the whole pass's cancel latency —
|
||||
* and every skip is reported via cbm_pipeline_add_file_error, never
|
||||
* silent. */
|
||||
static void doclink_scan_repo(cbm_pipeline_ctx_t *ctx, doclink_ctx_t *dc, const char *repo_path,
|
||||
const cbm_gbuf_node_t *const *files, int file_count, int *md_edges) {
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (cbm_pipeline_check_cancel(ctx)) {
|
||||
return;
|
||||
}
|
||||
if (!files[i]->file_path || !doclink_is_markdown_path(files[i]->file_path)) {
|
||||
continue;
|
||||
}
|
||||
@@ -439,12 +498,29 @@ static void doclink_scan_repo(doclink_ctx_t *dc, const char *repo_path,
|
||||
if (n <= 0 || (size_t)n >= sizeof(abs_path)) {
|
||||
continue;
|
||||
}
|
||||
char *source = doclink_read_file(abs_path);
|
||||
long file_size = 0;
|
||||
cbm_read_status_t rst = CBM_READ_OK;
|
||||
char *source = doclink_read_file(abs_path, &file_size, &rst);
|
||||
if (!source) {
|
||||
if (rst == CBM_READ_OVERSIZED) {
|
||||
long cap = cbm_max_file_bytes();
|
||||
char reason[96];
|
||||
(void)snprintf(reason, sizeof(reason), "oversized (%lld MB > %lld MB)",
|
||||
(long long)(file_size / (CBM_SZ_1K * CBM_SZ_1K)),
|
||||
(long long)(cap / (CBM_SZ_1K * CBM_SZ_1K)));
|
||||
cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, reason,
|
||||
"oversized");
|
||||
cbm_log_warn("doclinks.file_oversized", "path", files[i]->file_path);
|
||||
} else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) {
|
||||
cbm_pipeline_add_file_error(ctx->pipeline, files[i]->file_path, "read failed",
|
||||
"read");
|
||||
cbm_log_warn("doclinks.file_unreadable", "path", files[i]->file_path);
|
||||
}
|
||||
/* CBM_READ_EMPTY: benign 0-byte file, nothing to index, not reported. */
|
||||
continue;
|
||||
}
|
||||
int emitted = doclink_scan_file(dc, files[i], source);
|
||||
free(source);
|
||||
cbm_free(CBM_MEM_CLASS_EXTRACT, source);
|
||||
*md_edges += emitted;
|
||||
}
|
||||
}
|
||||
@@ -458,7 +534,7 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Early exit: no markdown/shell files means nothing to scan. */
|
||||
/* Early exit: no markdown files means nothing to scan. */
|
||||
if (!doclink_has_doc_files(files, file_count)) {
|
||||
cbm_log_info("doclinks.skip", "reason", "no_doc_files");
|
||||
return 0;
|
||||
@@ -483,7 +559,8 @@ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx) {
|
||||
}
|
||||
|
||||
int md_edges = 0;
|
||||
doclink_scan_repo(&dc, ctx->repo_path, files, file_count, &md_edges);
|
||||
doclink_scan_repo(ctx, &dc, ctx->repo_path, files, file_count, &md_edges);
|
||||
cbm_da_free(&dc.matches);
|
||||
cbm_ht_free(dc.files_by_path);
|
||||
|
||||
char buf1[CBM_SZ_16];
|
||||
|
||||
@@ -665,7 +665,7 @@ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project);
|
||||
/* Pre-dump pass: config ↔ code linking. */
|
||||
int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx);
|
||||
|
||||
/* Pre-dump pass: markdown/shell → file REFERENCES_FILE linking. */
|
||||
/* Pre-dump pass: markdown → file REFERENCES_FILE linking. */
|
||||
int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx);
|
||||
|
||||
/* Pre-dump pass: SIMILAR_TO edges via MinHash fingerprinting. */
|
||||
|
||||
+245
-1
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* test_doclinks.c — Tests for markdown/shell → file reference linking.
|
||||
* test_doclinks.c — Tests for markdown → file reference linking.
|
||||
*
|
||||
* Unit-test approach mirrors test_configlink.c: create real files in a
|
||||
* tmpdir (the pass reads content from disk), set up File nodes in a gbuf,
|
||||
@@ -294,6 +294,237 @@ TEST(doclinks_null_repo_path_skips) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Rooted "/x" resolves against the repo root ONLY ─────────────── */
|
||||
|
||||
TEST(doclinks_rooted_reference_skips_directory_join) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
/* docs/guide.md links "/README.md" (repo-root-relative by doc
|
||||
* convention). Both docs/README.md and the root README.md exist: a
|
||||
* rooted reference must resolve to the root file, never to a
|
||||
* same-named file in the referencing directory. */
|
||||
th_write_file(TH_PATH(fx.tmpdir, "docs/guide.md"), "See [home](/README.md).\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "docs/README.md"), "# Wrong target\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"), "# Root\n");
|
||||
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/guide.md");
|
||||
int64_t docs_readme_id = add_file_node(fx.gb, fx.project, "docs/README.md");
|
||||
int64_t root_readme_id = add_file_node(fx.gb, fx.project, "README.md");
|
||||
|
||||
run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
|
||||
ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, root_readme_id));
|
||||
ASSERT_NULL(find_ref_edge(fx.gb, doc_id, docs_readme_id));
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Explicit "./x" resolves against the referencing dir ONLY ────── */
|
||||
|
||||
TEST(doclinks_explicit_relative_no_root_fallback) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
/* docs/a.md links "./config.md". docs/config.md does not exist, but a
|
||||
* root config.md does — an explicit "./" reference must not fall back
|
||||
* to an unrelated same-named file at the repo root. */
|
||||
th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See [cfg](./config.md).\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "config.md"), "# Root config\n");
|
||||
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "docs/a.md");
|
||||
add_file_node(fx.gb, fx.project, "config.md");
|
||||
|
||||
int n = run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_EQ(n, 0);
|
||||
ASSERT_EQ(ref_edge_count(fx.gb), 0);
|
||||
(void)doc_id;
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── A line past the old 4 KiB copy cap is scanned in full ───────── */
|
||||
|
||||
TEST(doclinks_long_line_no_truncation) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
/* Padding + "src/foo.cpp" (no trailing newline) straddles the old
|
||||
* CBM_SZ_4K line-copy cap so that, under the old bug, the copy cut
|
||||
* exactly after "src/foo.c" and dropped "pp": a bare mention of
|
||||
* src/foo.cpp silently became a reference to the wrong file. Both
|
||||
* files exist so a wrong match is observable. */
|
||||
enum { FILLER_WORDS = 681 }; /* 681 * strlen("lorem ") == 4086 */
|
||||
char content[CBM_SZ_4K * 2];
|
||||
size_t pos = 0;
|
||||
for (int i = 0; i < FILLER_WORDS; i++) {
|
||||
memcpy(content + pos, "lorem ", 6);
|
||||
pos += 6;
|
||||
}
|
||||
ASSERT_EQ((int)pos, 4086);
|
||||
memcpy(content + pos, "src/foo.cpp", 11);
|
||||
pos += 11;
|
||||
content[pos] = '\0';
|
||||
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"), content);
|
||||
th_write_file(TH_PATH(fx.tmpdir, "src/foo.c"), "// wrong target\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "src/foo.cpp"), "// right target\n");
|
||||
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md");
|
||||
int64_t short_id = add_file_node(fx.gb, fx.project, "src/foo.c");
|
||||
int64_t long_id = add_file_node(fx.gb, fx.project, "src/foo.cpp");
|
||||
|
||||
run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
|
||||
ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, long_id));
|
||||
ASSERT_NULL(find_ref_edge(fx.gb, doc_id, short_id));
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── More than the old 256-target cap: nothing is dropped ────────── */
|
||||
|
||||
TEST(doclinks_many_distinct_targets_no_cap) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
enum { TARGETS = 300 }; /* > the old DOCLINK_MAX_REFS (256) */
|
||||
char content[TARGETS * 24];
|
||||
size_t pos = 0;
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "MANY.md");
|
||||
for (int i = 0; i < TARGETS; i++) {
|
||||
char rel[32];
|
||||
char line[40];
|
||||
snprintf(rel, sizeof(rel), "gen/f%03d.txt", i);
|
||||
th_write_file(TH_PATH(fx.tmpdir, rel), "x\n");
|
||||
add_file_node(fx.gb, fx.project, rel);
|
||||
int n = snprintf(line, sizeof(line), "%s\n", rel);
|
||||
memcpy(content + pos, line, (size_t)n);
|
||||
pos += (size_t)n;
|
||||
}
|
||||
content[pos] = '\0';
|
||||
th_write_file(TH_PATH(fx.tmpdir, "MANY.md"), content);
|
||||
|
||||
int n = run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_EQ(n, TARGETS);
|
||||
ASSERT_EQ(ref_edge_count(fx.gb), TARGETS);
|
||||
(void)doc_id;
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── ".." past the repo root is dropped, not resolved ────────────── */
|
||||
|
||||
TEST(doclinks_dotdot_escaping_root_dropped) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
/* README.md sits at the repo root (depth 0): "../outside.md" has
|
||||
* nowhere to go up from and must be dropped, not resolved. */
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"), "See [x](../outside.md).\n");
|
||||
|
||||
add_file_node(fx.gb, fx.project, "README.md");
|
||||
|
||||
int n = run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_EQ(n, 0);
|
||||
ASSERT_EQ(ref_edge_count(fx.gb), 0);
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── A bare URL in prose is not mistaken for a repo file path ────── */
|
||||
|
||||
TEST(doclinks_bare_url_in_prose_no_edge) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"),
|
||||
"Fetch it from http://host/path/file.c directly.\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "path/file.c"), "int x;\n");
|
||||
|
||||
add_file_node(fx.gb, fx.project, "README.md");
|
||||
add_file_node(fx.gb, fx.project, "path/file.c");
|
||||
|
||||
int n = run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_EQ(n, 0);
|
||||
ASSERT_EQ(ref_edge_count(fx.gb), 0);
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Exact-path matching: a bare basename never fuzzy-binds ──────── */
|
||||
|
||||
TEST(doclinks_backtick_basename_does_not_fuzzy_match) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
/* `main.c` in docs/ must resolve against docs/main.c or the repo-root
|
||||
* main.c ONLY (exact path match) — never against src/main.c just
|
||||
* because the basename matches. */
|
||||
th_write_file(TH_PATH(fx.tmpdir, "docs/a.md"), "See `main.c` for the entry point.\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n");
|
||||
|
||||
add_file_node(fx.gb, fx.project, "docs/a.md");
|
||||
add_file_node(fx.gb, fx.project, "src/main.c");
|
||||
|
||||
int n = run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_EQ(n, 0);
|
||||
ASSERT_EQ(ref_edge_count(fx.gb), 0);
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── CRLF line endings do not corrupt a matched target ───────────── */
|
||||
|
||||
TEST(doclinks_crlf_line_endings) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"),
|
||||
"# Title\r\nSee [build](scripts/build.sh) for details.\r\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "scripts/build.sh"), "#!/bin/sh\n");
|
||||
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md");
|
||||
int64_t script_id = add_file_node(fx.gb, fx.project, "scripts/build.sh");
|
||||
|
||||
run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
ASSERT_NOT_NULL(find_ref_edge(fx.gb, doc_id, script_id));
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Fenced code blocks are scanned like prose (pinned, not fixed) ── */
|
||||
|
||||
TEST(doclinks_fenced_code_block_bare_mention_pinned) {
|
||||
dl_fix_t fx;
|
||||
ASSERT_TRUE(dl_fix_init(&fx));
|
||||
|
||||
th_write_file(TH_PATH(fx.tmpdir, "README.md"),
|
||||
"Build it:\n\n```\ngcc src/main.c -o out\n```\n");
|
||||
th_write_file(TH_PATH(fx.tmpdir, "src/main.c"), "int main(void) { return 0; }\n");
|
||||
|
||||
int64_t doc_id = add_file_node(fx.gb, fx.project, "README.md");
|
||||
int64_t code_id = add_file_node(fx.gb, fx.project, "src/main.c");
|
||||
|
||||
run_doclinks(fx.gb, fx.project, fx.tmpdir);
|
||||
|
||||
const cbm_gbuf_edge_t *e = find_ref_edge(fx.gb, doc_id, code_id);
|
||||
ASSERT_NOT_NULL(e);
|
||||
ASSERT_TRUE(edge_has_strategy(e, "md_bare_mention"));
|
||||
ASSERT_NOT_NULL(strstr(e->properties_json, "\"confidence\":0.70"));
|
||||
|
||||
dl_fix_free(&fx);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Suite ───────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(doclinks) {
|
||||
@@ -309,4 +540,17 @@ SUITE(doclinks) {
|
||||
RUN_TEST(doclinks_resolves_relative_to_file_dir);
|
||||
RUN_TEST(doclinks_unresolvable_target_no_edge);
|
||||
RUN_TEST(doclinks_null_repo_path_skips);
|
||||
|
||||
/* Reference-kind precedence (rooted / explicit-relative / bare) */
|
||||
RUN_TEST(doclinks_rooted_reference_skips_directory_join);
|
||||
RUN_TEST(doclinks_explicit_relative_no_root_fallback);
|
||||
|
||||
/* Negative controls / regression guards */
|
||||
RUN_TEST(doclinks_long_line_no_truncation);
|
||||
RUN_TEST(doclinks_many_distinct_targets_no_cap);
|
||||
RUN_TEST(doclinks_dotdot_escaping_root_dropped);
|
||||
RUN_TEST(doclinks_bare_url_in_prose_no_edge);
|
||||
RUN_TEST(doclinks_backtick_basename_does_not_fuzzy_match);
|
||||
RUN_TEST(doclinks_crlf_line_endings);
|
||||
RUN_TEST(doclinks_fenced_code_block_bare_mention_pinned);
|
||||
}
|
||||
+1
-1
@@ -1182,7 +1182,7 @@ int main(int argc, char **argv) {
|
||||
/* Config link */
|
||||
RUN_SELECTED_SUITE(configlink);
|
||||
|
||||
/* Doc/shell file reference link */
|
||||
/* Markdown file reference link */
|
||||
RUN_SELECTED_SUITE(doclinks);
|
||||
|
||||
/* Infrastructure scanning */
|
||||
|
||||
@@ -314,6 +314,36 @@ TEST(pipeline_grpc_routes_cover_every_service_past_the_old_cap) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Pipeline-level: the doclinks pass runs as part of a real index and its
|
||||
* REFERENCES_FILE edges land in the store, not just in a unit test that
|
||||
* calls cbm_pipeline_pass_doclinks() directly. Registration-list gap: every
|
||||
* doclinks unit test calls the pass function by hand, so deleting its
|
||||
* {predump_doclinks, "doclinks", false} entry in pipeline.c would leave
|
||||
* every one of them green. */
|
||||
TEST(pipeline_doclinks_edge_lands_in_store) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_doclinks_pipeline_XXXXXX");
|
||||
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
|
||||
|
||||
write_temp_file(tmp, "README.md", "See [main](src/main.go) for the entry point.\n");
|
||||
write_temp_file(tmp, "src/main.go", "package main\n\nfunc main() {}\n");
|
||||
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/doclinks.db", tmp);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
|
||||
cbm_store_t *s = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(s);
|
||||
const char *project = cbm_pipeline_project_name(p);
|
||||
ASSERT_GT(cbm_store_count_edges_by_type(s, project, "REFERENCES_FILE"), 0);
|
||||
cbm_store_close(s);
|
||||
cbm_pipeline_free(p);
|
||||
th_rmtree(tmp);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Spilling must be invisible in the OUTPUT: the same repository indexed with
|
||||
* results parked on disk must produce the same graph as one indexed entirely in
|
||||
* memory. It did not. The namespace map that `use`/`using`/package imports
|
||||
@@ -15079,6 +15109,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(store_bulk_persistence);
|
||||
/* Integration: structure pass */
|
||||
RUN_TEST(pipeline_grpc_routes_cover_every_service_past_the_old_cap);
|
||||
RUN_TEST(pipeline_doclinks_edge_lands_in_store);
|
||||
RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory);
|
||||
RUN_TEST(pipeline_structure_nodes);
|
||||
RUN_TEST(pipeline_committed_counts_match_persisted);
|
||||
|
||||
Reference in new issue
Block a user