diff --git a/Makefile.cbm b/Makefile.cbm index 0c9eca047..9e1a7ac94 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -376,6 +376,8 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/doclink.c \ $(CBM_DIR)/doclink_cs.c \ $(CBM_DIR)/doclink_md.c \ + $(CBM_DIR)/doclink_rst.c \ + $(CBM_DIR)/doclink_py.c \ $(CBM_DIR)/doc_adr.c \ $(CBM_DIR)/doc_pdf.c \ $(CBM_DIR)/pdf/pdf_obj.c \ @@ -507,6 +509,7 @@ PIPELINE_SRCS = \ src/pipeline/doc_links_cs.c \ src/pipeline/doc_links_md.c \ src/pipeline/doc_links_pdf.c \ + src/pipeline/doc_links_rst.c \ src/pipeline/doc_links_msbuild.c \ src/pipeline/pass_route_nodes.c \ src/pipeline/pass_enrichment.c \ @@ -769,7 +772,7 @@ TEST_DISCOVER_SRCS = \ TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c -TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c +TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c TEST_WATCHER_SRCS = tests/test_watcher.c diff --git a/internal/cbm/doc_adr.c b/internal/cbm/doc_adr.c index 6913e340d..68ac3826a 100644 --- a/internal/cbm/doc_adr.c +++ b/internal/cbm/doc_adr.c @@ -1151,13 +1151,16 @@ static bool adr_ends_ci(const char *s, const char *suffix) { return n > k && adr_eq_ci(s + n - k, k, suffix); } -static bool adr_markdown_path(const char *rel) { - return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown"); +/* A Markdown or reStructuredText record (reST titles underlined with `=` or + * `-` read like setext headings). */ +static bool adr_document_path(const char *rel) { + return adr_ends_ci(rel, ".md") || adr_ends_ci(rel, ".mdx") || adr_ends_ci(rel, ".markdown") || + adr_ends_ci(rel, ".rst"); } void cbm_adr_extract(CBMExtractCtx *ctx) { if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0 || !ctx->rel_path || - !adr_markdown_path(ctx->rel_path)) { + !adr_document_path(ctx->rel_path)) { return; } const char *rel = ctx->rel_path; diff --git a/internal/cbm/doclink.c b/internal/cbm/doclink.c index a67364e34..1a3d06b3c 100644 --- a/internal/cbm/doclink.c +++ b/internal/cbm/doclink.c @@ -169,6 +169,12 @@ static const doclink_lang_t DOCLINK_LANGS[] = { .scope_tag = CBM_DOCLINK_CS_SCOPE_TAG, .portable_scope = cbm_doclink_cs_portable_scope}, {.lang = CBM_LANG_MARKDOWN, .scan_file = cbm_doclink_md_scan_file}, + {.lang = CBM_LANG_RST, .scan_file = cbm_doclink_rst_scan_file}, + /* Python: no doc references yet, only the scope the reST resolver reads + * (package re-exports, Sphinx conf.py settings) */ + {.lang = CBM_LANG_PYTHON, + .scan_scope = cbm_doclink_py_scan_scope, + .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG}, }; static const doclink_lang_t *doclink_lang(CBMLanguage lang) { @@ -181,7 +187,8 @@ static const doclink_lang_t *doclink_lang(CBMLanguage lang) { } bool cbm_doclink_lang_supported(CBMLanguage lang) { - return doclink_lang(lang) != NULL; + const doclink_lang_t *L = doclink_lang(lang); + return L && (L->parse_doc || L->scan_file); } /* ── Doc-line side map ───────────────────────────────────────────── */ @@ -402,6 +409,10 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) { L->scan_file(ctx); return; } + if (!L->parse_doc) { /* a scope, and no references (Python) */ + ctx->result->doc_scope = L->scan_scope ? L->scan_scope(ctx) : NULL; + return; + } int twin_count = 0; doclink_twin_key_t *twins = L->twin_label ? twin_keys(ctx, L->twin_of, &twin_count) : NULL; CBMDefArray *defs = &ctx->result->defs; diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h index ecdae767c..741b95aaa 100644 --- a/internal/cbm/doclink.h +++ b/internal/cbm/doclink.h @@ -64,26 +64,35 @@ * * The enum below and the family table (doclink.c) are both generated from this * list, so a value cannot exist without its name and its gate. */ -#define CBM_DOCLINK_FAMILY_LIST(X) \ - /* csharp */ \ - X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ - X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ - /* markdown (doclink_md.c): what a document's text names explicitly */ \ - X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ - /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ - X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ - /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ - /* names) only decides which line fragments a cross-line join replaces */ \ - X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ - X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ - /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ +#define CBM_DOCLINK_FAMILY_LIST(X) \ + /* csharp */ \ + X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \ + X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \ + /* markdown (doclink_md.c): what a document's text names explicitly */ \ + X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \ + /* an ADR's own "supersedes" / "replaces" statement (doc_adr.c): ADR -> ADR */ \ + X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \ + /* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \ + /* names) only decides which line fragments a cross-line join replaces */ \ + X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \ + X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \ + /* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \ + X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \ + X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \ + /* any language (CBM_LANG_COUNT): a URL is never an edge */ \ X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS") typedef enum { @@ -305,6 +314,25 @@ void cbm_adr_extract(CBMExtractCtx *ctx); /* The qualified-name tail of an ADR node: "" + this. */ #define CBM_ADR_QN_NAME "__adr__" +/* ── reStructuredText (doclink_rst.c) and Python's scope (doclink_py.c) ── */ + +/* The scan_file hook of a reStructuredText document: a Section per title + * (the text up to the next title as its docstring) and its references to + * code -- Sphinx roles and object directives, autodoc, include / + * literalinclude / kernel-doc paths, inline literals naming a path or a + * qualified name -- each with the section it is written in as its source. */ +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx); + +/* The scope blob of a Python file, which the reST resolver reads: a package + * `__init__.py`'s top-level explicit `from x import y [as z]` (what the + * package re-exports), and a Sphinx `conf.py`'s primary domain, extlink roles + * to repository paths and intersphinx names (read as text, never run). NULL + * for every other file. */ +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx); + +/* The Python scope blob starts with this tag line. */ +#define CBM_DOCLINK_PY_SCOPE_TAG "py1" + /* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's * text as docstring, property page) and the pages' structural code mentions * as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */ diff --git a/internal/cbm/doclink_py.c b/internal/cbm/doclink_py.c new file mode 100644 index 000000000..d762ab2e3 --- /dev/null +++ b/internal/cbm/doclink_py.c @@ -0,0 +1,419 @@ +/* + * doclink_py.c — the doc-link scope of a Python file: what the reST resolver + * (src/pipeline/doc_links_rst.c) needs from Python sources to resolve Sphinx + * references the way Sphinx does. + * + * I \t level \t module \t name \t alias a package `__init__.py`'s top-level + * `from module import name [as alias]`: + * the package re-exports `alias` + * (`django.db.models.Model` is + * django/db/models/base.py's Model). + * Star imports are not read (the field + * test's resolver did not follow them). + * C the file is a `conf.py`: documents in + * and below its directory are one + * Sphinx documentation set + * D \t domain its primary_domain + * X \t role an extlink role whose URL is a + * repository blob/tree path + * (:source:`django/db/x.py`) + * S \t name an intersphinx name (`python:` in + * :class:`python:dict` is external) + * + * conf.py is read as text, with the field test's patterns, and never run. + * Every other Python file has no scope (NULL), so a change to it never + * changes another file's resolution through this blob. + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" + +#include +#include +#include +#include + +enum { + PY_SCOPE_FIELD_MAX = 256, /* a module, name or setting longer than this is skipped */ + PY_SB_INIT = 256, + PY_INT_DIGITS = 16, +}; + +/* A small arena string builder (the blob lives in the result arena). */ +typedef struct { + CBMArena *a; + char *buf; + size_t len; + size_t cap; + bool failed; +} py_sb_t; + +static void py_put(py_sb_t *sb, const char *s, size_t n) { + if (sb->failed) { + return; + } + if (sb->len + n + 1 > sb->cap) { + size_t ncap = sb->cap ? sb->cap : PY_SB_INIT; + while (ncap < sb->len + n + 1) { + ncap *= 2; + } + char *grown = (char *)cbm_arena_alloc(sb->a, ncap); + if (!grown) { + sb->failed = true; + return; + } + if (sb->len > 0) { + memcpy(grown, sb->buf, sb->len); + } + sb->buf = grown; + sb->cap = ncap; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +/* One record: its kind, then each field after a TAB, then a newline. */ +static void py_rec(py_sb_t *sb, const char *kind, int nfields, const char *const *f, + const size_t *fl) { + py_put(sb, kind, strlen(kind)); + for (int i = 0; i < nfields; i++) { + py_put(sb, "\t", 1); + py_put(sb, f[i], fl[i]); + } + py_put(sb, "\n", 1); +} + +static const char *py_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + 1 : path; +} + +static bool py_field_ok(const char *s, size_t n) { + if (n == 0 || n > PY_SCOPE_FIELD_MAX) { + return false; + } + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if (c == '\t' || c == '\n' || c == '\r' || c < ' ') { + return false; + } + } + return true; +} + +static void py_text(const CBMExtractCtx *ctx, TSNode n, const char **s, size_t *len) { + uint32_t a = ts_node_start_byte(n); + uint32_t b = ts_node_end_byte(n); + if (b > (uint32_t)ctx->source_len || a > b) { + *s = ""; + *len = 0; + return; + } + *s = ctx->source + a; + *len = b - a; +} + +/* One `from module import names` statement of the package's top level. */ +static void py_from_import(const CBMExtractCtx *ctx, TSNode stmt, py_sb_t *sb) { + TSNode mod = ts_node_child_by_field_name(stmt, "module_name", (uint32_t)strlen("module_name")); + if (ts_node_is_null(mod)) { + return; + } + int level = 0; + const char *ms = ""; + size_t ml = 0; + if (strcmp(ts_node_type(mod), "relative_import") == 0) { + uint32_t nc = ts_node_child_count(mod); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(mod, i); + const char *k = ts_node_type(c); + if (strcmp(k, "import_prefix") == 0) { + const char *ps; + size_t pl; + py_text(ctx, c, &ps, &pl); + for (size_t j = 0; j < pl; j++) { + level += ps[j] == '.'; + } + } else if (strcmp(k, "dotted_name") == 0) { + py_text(ctx, c, &ms, &ml); + } + } + } else { + py_text(ctx, mod, &ms, &ml); + } + if (ml > 0 && !py_field_ok(ms, ml)) { + return; + } + uint32_t nc = ts_node_child_count(stmt); + for (uint32_t i = 0; i < nc; i++) { + const char *field = ts_node_field_name_for_child(stmt, i); + if (!field || strcmp(field, "name") != 0) { + continue; + } + TSNode c = ts_node_child(stmt, i); + TSNode name = c; + TSNode alias = c; + if (strcmp(ts_node_type(c), "aliased_import") == 0) { + name = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name")); + alias = ts_node_child_by_field_name(c, "alias", (uint32_t)strlen("alias")); + if (ts_node_is_null(name) || ts_node_is_null(alias)) { + continue; + } + } + const char *ns; + size_t nl; + const char *as; + size_t al; + py_text(ctx, name, &ns, &nl); + py_text(ctx, alias, &as, &al); + if (!py_field_ok(ns, nl) || !py_field_ok(as, al)) { + continue; + } + char lv[PY_INT_DIGITS]; + int lvn = snprintf(lv, sizeof(lv), "%d", level); + const char *f[] = {lv, ms, ns, as}; + const size_t fl[] = {lvn > 0 ? (size_t)lvn : 0, ml, nl, al}; + py_rec(sb, "I", 4, f, fl); + } +} + +/* ── conf.py, read as text (the field test's patterns) ───────────── */ + +static bool py_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool py_word(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +static size_t py_skip_ws(const char *s, size_t n, size_t i) { + while (i < n && py_ws(s[i])) { + i++; + } + return i; +} + +/* `^name\s*=\s*` at a line start: the index after it, or 0. */ +static size_t py_assign_at(const char *s, size_t n, size_t line, const char *name) { + size_t nl = strlen(name); + if (line + nl > n || memcmp(s + line, name, nl) != 0) { + return 0; + } + size_t i = py_skip_ws(s, n, line + nl); + if (i >= n || s[i] != '=') { + return 0; + } + return py_skip_ws(s, n, i + 1); +} + +/* conf_scalar: ^name\s*=\s*["']([^"']*)["'] (first match). */ +static bool py_conf_scalar(const char *s, size_t n, const char *name, const char **v, size_t *vl) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && (s[i] == '"' || s[i] == '\'')) { + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j < n) { + *v = s + i + 1; + *vl = j - i - 1; + return true; + } + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* conf_dict_block: ^name\s*=\s*\{(.*?)^\} -- the text between, or false. */ +static bool py_conf_block(const char *s, size_t n, const char *name, size_t *a, size_t *b) { + for (size_t line = 0; line < n;) { + size_t i = py_assign_at(s, n, line, name); + if (i > 0 && i < n && s[i] == '{') { + size_t k = i + 1; + while (k < n) { + if (s[k] == '}' && (k == 0 || s[k - 1] == '\n')) { + *a = i + 1; + *b = k; + return true; + } + k++; + } + return false; + } + const char *nl = memchr(s + line, '\n', n - line); + line = nl ? (size_t)(nl - s) + 1 : n; + } + return false; +} + +/* ["']([\w.-]+)["'] at s[i] (extra: also '.'): its end after the closing quote, or 0. */ +static size_t py_quoted_key(const char *s, size_t n, size_t i, bool dots, size_t *ka, size_t *kb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return 0; + } + size_t j = i + 1; + while (j < n && (py_word(s[j]) || s[j] == '-' || (dots && s[j] == '.'))) { + j++; + } + if (j == i + 1 || j >= n || (s[j] != '"' && s[j] != '\'')) { + return 0; + } + *ka = i + 1; + *kb = j; + return j + 1; +} + +/* ["']([^"']+)["'] at s[i]: the value span, or false. */ +static bool py_quoted_value(const char *s, size_t n, size_t i, size_t *va, size_t *vb) { + if (i >= n || (s[i] != '"' && s[i] != '\'')) { + return false; + } + size_t j = i + 1; + while (j < n && s[j] != '"' && s[j] != '\'') { + j++; + } + if (j == i + 1 || j >= n) { + return false; + } + *va = i + 1; + *vb = j; + return true; +} + +/* /(blob|tree)/[^/]+/%s in the URL: a repository path pattern. */ +static bool py_repo_url(const char *u, size_t n) { + static const char *const kinds[] = {"/blob/", "/tree/", NULL}; + for (int k = 0; kinds[k]; k++) { + size_t kl = strlen(kinds[k]); + for (size_t i = 0; i + kl <= n; i++) { + if (memcmp(u + i, kinds[k], kl) != 0) { + continue; + } + size_t j = i + kl; + size_t ref = j; + while (j < n && u[j] != '/') { + j++; + } + if (j > ref && j + 3 <= n && memcmp(u + j, "/%s", 3) == 0) { + return true; + } + } + } + return false; +} + +static void py_extlink(py_sb_t *sb, const char *s, size_t ka, size_t kb, size_t va, size_t vb) { + if (py_repo_url(s + va, vb - va) && py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "X", 1, f, fl); + } +} + +static void py_conf(const char *s, size_t n, py_sb_t *sb) { + py_rec(sb, "C", 0, NULL, NULL); + const char *v; + size_t vl; + if (py_conf_scalar(s, n, "primary_domain", &v, &vl) && py_field_ok(v, vl)) { + const char *f[] = {v}; + const size_t fl[] = {vl}; + py_rec(sb, "D", 1, f, fl); + } + size_t a; + size_t b; + if (py_conf_block(s, n, "intersphinx_mapping", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, true, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_field_ok(s + ka, kb - ka)) { + const char *f[] = {s + ka}; + const size_t fl[] = {kb - ka}; + py_rec(sb, "S", 1, f, fl); + } + i = e - 1; + } + } + /* extlinks = {'role': ('url%s', ...)} and extlinks['role'] = ('url%s', ...) */ + if (py_conf_block(s, n, "extlinks", &a, &b)) { + for (size_t i = a; i < b; i++) { + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, b, i, false, &ka, &kb); + if (e == 0) { + continue; + } + size_t c = py_skip_ws(s, b, e); + size_t va; + size_t vb; + if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' && + py_quoted_value(s, b, py_skip_ws(s, b, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + i = e - 1; + } + } + static const char key_open[] = "extlinks["; + size_t kol = strlen(key_open); + for (size_t at = 0; at + kol <= n; at++) { + if (memcmp(s + at, key_open, kol) != 0) { + continue; + } + size_t i = at + kol; + size_t ka; + size_t kb; + size_t e = py_quoted_key(s, n, i, false, &ka, &kb); + if (e == 0 || e >= n || s[e] != ']') { + continue; + } + size_t c = py_skip_ws(s, n, e + 1); + size_t va; + size_t vb; + if (c < n && s[c] == '=' && (c = py_skip_ws(s, n, c + 1)) < n && s[c] == '(' && + py_quoted_value(s, n, py_skip_ws(s, n, c + 1), &va, &vb)) { + py_extlink(sb, s, ka, kb, va, vb); + } + } +} + +const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx) { + if (!ctx || !ctx->rel_path || !ctx->source) { + return NULL; + } + const char *base = py_base(ctx->rel_path); + bool init = strcmp(base, "__init__.py") == 0; + bool conf = strcmp(base, "conf.py") == 0; + if (!init && !conf) { + return NULL; + } + py_sb_t sb = {.a = ctx->arena}; + py_put(&sb, CBM_DOCLINK_PY_SCOPE_TAG "\n", strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n")); + if (conf) { + py_conf(ctx->source, (size_t)ctx->source_len, &sb); + } + if (init && !ts_node_is_null(ctx->root)) { + uint32_t nc = ts_node_child_count(ctx->root); + for (uint32_t i = 0; i < nc; i++) { + TSNode c = ts_node_child(ctx->root, i); + if (strcmp(ts_node_type(c), "import_from_statement") == 0) { + py_from_import(ctx, c, &sb); + } + } + } + if (sb.failed) { + ctx->result->doc_links.failed = true; + return NULL; + } + return sb.buf; +} diff --git a/internal/cbm/doclink_rst.c b/internal/cbm/doclink_rst.c new file mode 100644 index 000000000..a50498f14 --- /dev/null +++ b/internal/cbm/doclink_rst.c @@ -0,0 +1,1791 @@ +/* + * doclink_rst.c — reStructuredText documents: their sections, and the code + * their text names (the extraction half; src/pipeline/doc_links_rst.c + * resolves). + * + * Structure: the field test's line model (H3 RstDoc) with CPython's str + * semantics -- tabs expanded to 8 columns, str.strip / str.split whitespace + * and re's \s and \w from the Unicode tables the PDF scanner uses. Every line + * gets a kind: title, adornment, prose, directive head / argument / option, + * literal block (after `::` or in a code directive), comment, hyperlink + * target. Titles are found by adornment style (over+under or under only; the + * level is the style's first appearance). Each title becomes a Section + * definition: name the title, docstring the text up to the next title, lines + * from its title to the line before the next one. Text before the first title + * belongs to the file. + * + * References become tokens whose `raw` is the reference AS WRITTEN followed by + * TAB-separated fields the resolver reads (a doc_link_unresolved row shows the + * written form first; whitespace inside a field is collapsed, so no field + * holds a TAB): + * role :py:class:`x`, :func:`~a.b`, :c:type:`t`, :source:`p` + * written, role, content, module, class, C namespace + * object .. py:class:: sig, .. function:: sig (one per signature) + * written, directive, signature, module, class, C namespace, + * the Python full name in its class ("" when Sphinx's + * signature grammar does not read it) + * autodoc .. autoclass:: a.b + * written, directive, argument, module, class + * include .. include:: / .. kernel-include:: of a code file + * written, directive, path + * literalinclude .. literalinclude:: path + * written, path, :lines:, :pyobject:, "1" when a + * :start-after: / :end-before: / :start-at: / :end-at: + * bounds it by text (the resolver binds the file then) + * kernel_doc .. kernel-doc:: path, and each name of :identifiers: / + * :functions: -- written, path, name ("" for the file) + * code_path, an inline literal ``...`` that names a path or a qualified + * code_name name (the Markdown code-span classifier); raw is its text + * A role or directive without a domain is taken for Python's when this file + * sets contexts (`module`, `currentmodule`, class directives: Sphinx's + * default domain); which domain it belongs to is the resolver's call (the + * documentation set's conf.py). Roles of no code domain are tokens only when + * their content is a path (an extlink such as :source:`django/db/x.py`). + */ +#include "doclink.h" + +#include "arena.h" +#include "cbm.h" +#include "helpers.h" +#include "pdf/pdf_unicode.h" + +#include +#include +#include +#include + +enum { + RST_TAB = 8, + RST_SHORT_ADORN = 4, /* an underline this long fits any title; a transition */ + RST_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */ + RST_SPAN_MAX = 150, /* an inline literal the span classifier reads (MD_SPAN_MAX) */ + RST_INIT_CAP = 16, + RST_REPLACEMENT = 0xFFFD, + RST_ASCII_END = 0x80, + RST_LEAD2_MIN = 0xC2, /* UTF-8 lead bytes (no overlong two-byte form) */ + RST_LEAD3_MIN = 0xE0, + RST_LEAD3_MAX = 0xEF, + RST_LEAD4_MIN = 0xF0, + RST_LEAD4_MAX = 0xF4, + RST_NEED3 = 2, + RST_NEED4 = 3, + RST_BULLET = 0x2022, + RST_SIG_ARROW = 2, /* "->" */ +}; + +#define RST_SEP '\t' + +typedef enum { + K_BLANK = 0, + K_TITLE, + K_ADORN, + K_DHEAD, + K_DHEAD_PROSE, + K_DARG, + K_DARG_PROSE, + K_DOPT, + K_LITERAL, + K_PLIT, + K_SKIP, + K_TARGET, + K_COMMENT, + K_PROSE, +} rst_kind_t; + +typedef struct { + char *s; /* the line, tabs expanded, NUL-terminated */ + int len; /* bytes */ + int indent; /* leading spaces (H3 indent_of) */ + bool blank; /* nothing but whitespace */ +} rst_line_t; + +typedef struct { + const char *key; + const char *val; +} rst_opt_t; + +typedef struct { + int line; + int indent; + const char *name; /* as written, lower-cased, with its domain */ + const char *dom; /* "py", "c", "std" or the written domain */ + const char *dname; /* without the domain */ + const char **args; + int nargs; + rst_opt_t *opts; + int nopts; + int cs; /* content lines [cs, ce) */ + int ce; +} rst_dir_t; + +typedef struct { + int first; /* the title block's first line (an overline, or the title) */ + int idx; /* the title text's line */ + int next; /* the line after the title block */ + const char *text; + const char *qn; + bool shared_qn; /* another title of the file has the same QN: the file is the source */ +} rst_title_t; + +typedef struct { + int line; /* 0-based */ + int col; /* byte offset in its line; directives use their indent */ + int syntax; + const char *raw; +} rst_tok_t; + +typedef struct { + CBMExtractCtx *ctx; + CBMArena *scratch; + rst_line_t *L; + int n; + unsigned char *K; + rst_dir_t *dirs; + int ndirs; + int cap_dirs; + rst_title_t *titles; + int ntitles; + int cap_titles; + char styles[2 * RST_INIT_CAP * RST_INIT_CAP]; /* (char, overline) pairs in first-seen order */ + int nstyles; + const char **mod_at; + const char **cls_at; + const char **ns_at; + rst_tok_t *toks; + int ntoks; + int cap_toks; + char *span_buf; + bool failed; +} rst_doc_t; + +/* ── Unicode, as CPython's str and re see it ─────────────────────── */ + +/* The code point at s[i] and its length; a malformed byte is U+FFFD of one + * byte (decode(errors='replace')). */ +static uint32_t rst_cp(const char *s, int len, int i, int *n) { + unsigned char c = (unsigned char)s[i]; + if (c < RST_ASCII_END) { + *n = 1; + return c; + } + int need = 0; /* continuation bytes of a well-formed lead byte */ + if (c >= RST_LEAD4_MIN && c <= RST_LEAD4_MAX) { + need = RST_NEED4; + } else if (c >= RST_LEAD3_MIN && c <= RST_LEAD3_MAX) { + need = RST_NEED3; + } else if (c >= RST_LEAD2_MIN && c < RST_LEAD3_MIN) { + need = 1; + } + if (need == 0 || i + need >= len) { + *n = 1; + return RST_REPLACEMENT; + } + uint32_t cp = c & (0x3F >> need); + for (int k = 1; k <= need; k++) { + unsigned char cc = (unsigned char)s[i + k]; + if ((cc & 0xC0) != 0x80) { + *n = 1; + return RST_REPLACEMENT; + } + cp = (cp << 6) | (cc & 0x3F); + } + static const uint32_t min_cp[] = {0, 0x80, 0x800, 0x10000}; + if (cp < min_cp[need] || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) { + *n = 1; + return RST_REPLACEMENT; + } + *n = need + 1; + return cp; +} + +static bool rst_space_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return cp == ' ' || (cp >= '\t' && cp <= '\r') || (cp >= 0x1C && cp <= 0x1F); + } + return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, cp); +} + +static bool rst_word_cp(uint32_t cp) { + if (cp < RST_ASCII_END) { + return (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z') || (cp >= '0' && cp <= '9') || + cp == '_'; + } + return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, cp); +} + +static bool rst_alpha_ascii(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool rst_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +/* Whitespace code points from s[i]: the index after them. */ +static int rst_skip_ws(const char *s, int len, int i) { + while (i < len) { + int n; + if (!rst_space_cp(rst_cp(s, len, i, &n))) { + break; + } + i += n; + } + return i; +} + +/* str.strip(): [*a, *b) without leading and trailing whitespace. */ +static void rst_strip(const char *s, int len, int *a, int *b) { + int i = rst_skip_ws(s, len, 0); + int end = i; + int k = i; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + k += n; + if (!rst_space_cp(cp)) { + end = k; + } + } + *a = i; + *b = end; +} + +/* Code points in s[a, b). */ +static int rst_cps(const char *s, int a, int b) { + int count = 0; + while (a < b) { + int n; + (void)rst_cp(s, b, a, &n); + a += n; + count++; + } + return count; +} + +/* ' '.join(s.split()), into the arena. */ +static char *rst_collapse(CBMArena *a, const char *s, int len) { + char *out = (char *)cbm_arena_alloc(a, (size_t)len + 1); + if (!out) { + return NULL; + } + int w = 0; + int i = 0; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + out[w] = '\0'; + return out; +} + +/* ── Growing arrays in the scratch arena ─────────────────────────── */ + +static void *rst_grow(rst_doc_t *d, void *items, int count, int *cap, size_t size) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * 2 : RST_INIT_CAP; + void *neu = cbm_arena_alloc(d->scratch, (size_t)ncap * size); + if (!neu) { + d->failed = true; + return NULL; + } + if (items && count) { + memcpy(neu, items, (size_t)count * size); + } + *cap = ncap; + return neu; +} + +/* ── Lines ───────────────────────────────────────────────────────── */ + +/* text.expandtabs(8).split('\n'): the column restarts after '\r' and '\n'. */ +static bool rst_lines(rst_doc_t *d, const char *src, int n) { + int count = 1; + for (int i = 0; i < n; i++) { + count += src[i] == '\n'; + } + d->L = (rst_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L)); + d->K = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count); + if (!d->L || !d->K) { + return false; + } + memset(d->K, 0, (size_t)count); + int pos = 0; + for (int li = 0; li < count; li++) { + const char *nl = memchr(src + pos, '\n', (size_t)(n - pos)); + int end = nl ? (int)(nl - src) : n; + int tabs = 0; + for (int i = pos; i < end; i++) { + tabs += src[i] == '\t'; + } + int cap = (end - pos) + tabs * RST_TAB + 1; + char *out = (char *)cbm_arena_alloc(d->scratch, (size_t)cap); + if (!out) { + return false; + } + int w = 0; + int col = 0; + int i = pos; + while (i < end) { + int cn; + uint32_t cp = rst_cp(src, end, i, &cn); + if (cp == '\t') { + int fill = RST_TAB - (col % RST_TAB); + memset(out + w, ' ', (size_t)fill); + w += fill; + col += fill; + } else { + memcpy(out + w, src + i, (size_t)cn); + w += cn; + col = cp == '\r' ? 0 : col + 1; + } + i += cn; + } + out[w] = '\0'; + rst_line_t *ln = &d->L[li]; + ln->s = out; + ln->len = w; + int sp = 0; + while (sp < w && out[sp] == ' ') { + sp++; + } + ln->indent = sp; + int a; + int b; + rst_strip(out, w, &a, &b); + ln->blank = a >= b; + pos = end + 1; + } + d->n = count; + return true; +} + +/* ── Line patterns (H3's regexes) ────────────────────────────────── */ + +static bool rst_punct(unsigned char c) { + return (c >= '!' && c <= '/') || (c >= ':' && c <= '@') || (c >= '[' && c <= '`') || + (c >= '{' && c <= '~'); +} + +/* ADORN_RE ^([!-/:-@\[-`{-~])\1*\s*$ */ +static bool rst_adorn(const char *s, int len) { + if (len == 0 || !rst_punct((unsigned char)s[0])) { + return false; + } + int i = 1; + while (i < len && s[i] == s[0]) { + i++; + } + return rst_skip_ws(s, len, i) == len; +} + +/* s.rstrip() length. */ +static int rst_rlen(const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + return a >= b ? 0 : b; +} + +static bool rst_style_index(rst_doc_t *d, char ch, bool over, int *level) { + for (int i = 0; i < d->nstyles; i++) { + if (d->styles[2 * i] == ch && d->styles[(2 * i) + 1] == (char)over) { + *level = i; + return true; + } + } + if ((size_t)(2 * (d->nstyles + 1)) > sizeof(d->styles)) { + return false; + } + d->styles[2 * d->nstyles] = ch; + d->styles[(2 * d->nstyles) + 1] = (char)over; + *level = d->nstyles++; + return true; +} + +/* H3 _title_at: the title's line, its style and the line after the block. */ +static bool rst_title_at(const rst_doc_t *d, int i, int *tidx, char *ch, bool *over, int *next) { + const rst_line_t *L = d->L; + int n = d->n; + int sl = rst_rlen(L[i].s, L[i].len); + int sa; + int sb; + rst_strip(L[i].s, sl, &sa, &sb); + if (rst_adorn(L[i].s, sl)) { + if (sb - sa == 2 && L[i].s[sa] == ':' && L[i].s[sa + 1] == ':') { + return false; + } + if (i + 2 < n) { + const rst_line_t *t = &L[i + 1]; + int ul = rst_rlen(L[i + 2].s, L[i + 2].len); + const char *u = L[i + 2].s; + if (!t->blank && !rst_adorn(t->s, t->len) && rst_adorn(u, ul) && u[0] == L[i].s[sa]) { + *tidx = i + 1; + *ch = L[i].s[sa]; + *over = true; + *next = i + 3; + return true; + } + } + return false; + } + if (i + 1 < n && !(sl >= 2 && L[i].s[0] == '.' && L[i].s[1] == '.')) { + const char *u = L[i + 1].s; + int ul = rst_rlen(u, L[i + 1].len); + if (ul > 0 && !rst_space_cp((unsigned char)u[0]) && rst_adorn(u, ul)) { + int ua; + int ub; + rst_strip(u, ul, &ua, &ub); + bool dcolon = ub - ua == 2 && u[ua] == ':' && u[ua + 1] == ':'; + int ulen = ub - ua; /* ASCII punctuation: bytes are code points */ + if (!dcolon && (ulen >= rst_cps(L[i].s, sa, sb) || ulen >= RST_SHORT_ADORN)) { + *tidx = i; + *ch = u[ua]; + *over = false; + *next = i + 2; + return true; + } + } + } + return false; +} + +static void rst_mark(rst_doc_t *d, int a, int b, rst_kind_t kind) { + for (int k = a; k < b; k++) { + d->K[k] = (unsigned char)(d->L[k].blank ? K_BLANK : kind); + } +} + +static int rst_block_end(const rst_doc_t *d, int j, int ind) { + int k = j; + while (k < d->n && (d->L[k].blank || d->L[k].indent > ind)) { + k++; + } + return k; +} + +static bool rst_name_char(const char *s, int len, int i, int *n) { + uint32_t cp = rst_cp(s, len, i, n); + return cp == ':' || cp == '.' || cp == '+' || cp == '-' || rst_word_cp(cp); +} + +/* DIRECTIVE_RE: ^(\s*)\.\.\s+(?:\|[^|]+\|\s+)?([A-Za-z0-9][\w:.+-]*?)::(?:\s+(.*?))?\s*$ + * Fills the indent (code points), the name span and the first argument + * span (stripped). */ +static bool rst_directive_match(const char *s, int len, int *ind, int *na, int *nb, int *aa, + int *ab) { + int p = rst_skip_ws(s, len, 0); + *ind = rst_cps(s, 0, p); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + int q = rst_skip_ws(s, len, p + 2); + if (q == p + 2) { + return false; + } + p = q; + if (p < len && s[p] == '|') { + const char *bar = memchr(s + p + 1, '|', (size_t)(len - p - 1)); + if (bar && bar > s + p + 1) { + int after = (int)(bar - s) + 1; + int w = rst_skip_ws(s, len, after); + if (w > after) { + p = w; + } + } + } + if (p >= len || !(rst_alpha_ascii((unsigned char)s[p]) || rst_digit((unsigned char)s[p]))) { + return false; + } + int e = p + 1; + for (;;) { + if (e + 2 <= len && s[e] == ':' && s[e + 1] == ':') { + int t = e + 2; + int w = rst_skip_ws(s, len, t); + if (t == len || w > t) { + *na = p; + *nb = e; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *aa = w + a; + *ab = w + (b > a ? b : a); + return true; + } + } + int cn; + if (e >= len || !rst_name_char(s, len, e, &cn)) { + return false; + } + e += cn; + } +} + +/* EXPLICIT_RE ^(\s*)\.\.(?:\s|$) */ +static bool rst_explicit_match(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p + 2 > len || s[p] != '.' || s[p + 1] != '.') { + return false; + } + if (p + 2 == len) { + return true; + } + int n; + return rst_space_cp(rst_cp(s, len, p + 2, &n)); +} + +/* OPTION_LINE_RE ^(\s*):([A-Za-z][\w-]*):(?:\s+(.*?))?\s*$ */ +static bool rst_option_match(const char *s, int len, int *ka, int *kb, int *va, int *vb) { + int p = rst_skip_ws(s, len, 0); + if (p >= len || s[p] != ':' || p + 1 >= len || !rst_alpha_ascii((unsigned char)s[p + 1])) { + return false; + } + int k = p + 2; + while (k < len) { + int n; + uint32_t cp = rst_cp(s, len, k, &n); + if (cp != '-' && !rst_word_cp(cp)) { + break; + } + k += n; + } + if (k >= len || s[k] != ':') { + return false; + } + int t = k + 1; + int w = rst_skip_ws(s, len, t); + if (t != len && w == t) { + return false; + } + *ka = p + 1; + *kb = k; + int a; + int b; + rst_strip(s + w, len - w, &a, &b); + *va = w + a; + *vb = w + (b > a ? b : a); + return true; +} + +static bool rst_roman(unsigned char c) { + return strchr("ivxlcdmIVXLCDM", c) != NULL && c != '\0'; +} + +/* LIST_MARKER_RE: the match length in code points, or -1. */ +static int rst_list_marker(const char *s, int len) { + int p = rst_skip_ws(s, len, 0); + if (p >= len) { + return -1; + } + int n; + uint32_t cp = rst_cp(s, len, p, &n); + int q = -1; + if (cp == '-' || cp == '*' || cp == '+' || cp == RST_BULLET) { + q = p + n; + } else { + int r = p + (s[p] == '('); + int e = -1; + if (r < len && rst_digit((unsigned char)s[r])) { + e = r; + while (e < len && rst_digit((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e < 0 && r + 1 < len && rst_alpha_ascii((unsigned char)s[r]) && + (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r + 1 < len && s[r] == '#' && (s[r + 1] == '.' || s[r + 1] == ')')) { + e = r + 1; + } + if (e < 0 && r < len && rst_roman((unsigned char)s[r])) { + e = r; + while (e < len && rst_roman((unsigned char)s[e])) { + e++; + } + if (!(e < len && (s[e] == '.' || s[e] == ')'))) { + e = -1; + } + } + if (e >= 0) { + q = e + 1; + } + } + if (q < 0) { + return -1; + } + int w = rst_skip_ws(s, len, q); + if (w == q || w >= len) { + return -1; + } + return rst_cps(s, 0, w); +} + +/* ── Block structure (H3 _scan) ──────────────────────────────────── */ + +static const char *const PY_DOMAIN_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", "currentmodule", NULL}; +static const char *const LITERAL_CONTENT[] = { + "code-block", "code", "sourcecode", "literalinclude", "raw", "math", + "graphviz", "digraph", "graph", "productionlist", "console", "doctest", + "testcode", "testoutput", "testsetup", "testcleanup", "highlight", "kernel-render", + "prompt", "kernel-include", "include", "kernel-abi", "kernel-feat", "uml", + "ipython", "mermaid", NULL}; +static const char *const SKIP_CONTENT[] = {"toctree", "index", "tabularcolumns", NULL}; +static const char *const ARG_PROSE[] = {"note", "warning", "tip", + "hint", "important", "caution", + "attention", "danger", "error", + "seealso", "admonition", "versionadded", + "versionchanged", "deprecated", "versionremoved", + "rubric", "topic", "sidebar", + "replace", "centered", NULL}; +static const char *const PY_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", NULL}; +static const char *const C_OBJ[] = {"function", "macro", "type", "struct", "union", + "enum", "enumerator", "member", "var", NULL}; +static const char *const AUTODOC[] = { + "automodule", "autoclass", "autofunction", "automethod", "autoattribute", + "autoexception", "autodata", "autodecorator", "autoproperty", NULL}; +static const char *const PY_ROLES[] = {"func", "class", "meth", "attr", "mod", "exc", + "data", "obj", "const", "deco", NULL}; +static const char *const C_ROLES[] = {"func", "macro", "struct", "union", "enum", "enumerator", + "type", "member", "data", "var", NULL}; +/* include / kernel-include of these is code (H3 CODE_EXT) */ +static const char *const CODE_EXT[] = {".c", ".h", ".s", ".py", ".sh", ".rs", ".pl", + ".awk", ".cpp", ".cc", ".hpp", ".go", ".js", ".ts", + ".lds", ".dts", ".dtsi", ".yaml", ".yml", ".json", ".toml", + ".mk", ".cmake", ".bpf", ".tc", NULL}; + +static bool rst_in(const char *s, const char *const *list) { + for (int i = 0; s && list[i]; i++) { + if (strcmp(s, list[i]) == 0) { + return true; + } + } + return false; +} + +/* H3 split_domain with Python as the primary domain. */ +static void rst_split_domain(rst_doc_t *d, const char *name, const char *const *py_members, + const char **dom, const char **rest) { + const char *colon = strchr(name, ':'); + if (colon) { + *dom = cbm_arena_strndup(d->scratch, name, (size_t)(colon - name)); + *rest = colon + 1; + if (!*dom) { + d->failed = true; + *dom = ""; + } + return; + } + *dom = rst_in(name, py_members) ? "py" : "std"; + *rest = name; +} + +static char *rst_dup_stripped(rst_doc_t *d, const char *s, int len) { + int a; + int b; + rst_strip(s, len, &a, &b); + char *out = cbm_arena_strndup(d->scratch, s + a, (size_t)(b > a ? b - a : 0)); + if (!out) { + d->failed = true; + } + return out; +} + +static int rst_directive(rst_doc_t *d, int i, int ind, int na, int nb, int aa, int ab) { + rst_line_t *L = d->L; + int n = d->n; + char *name = cbm_arena_strndup(d->scratch, L[i].s + na, (size_t)(nb - na)); + if (!name) { + d->failed = true; + return n; + } + for (char *c = name; *c; c++) { + if (*c >= 'A' && *c <= 'Z') { + *c = (char)(*c + ('a' - 'A')); + } + } + rst_dir_t dir = {.line = i, .indent = ind, .name = name}; + rst_split_domain(d, name, PY_DOMAIN_OBJ, &dir.dom, &dir.dname); + bool prose_arg = rst_in(dir.dname, ARG_PROSE); + d->K[i] = prose_arg ? K_DHEAD_PROSE : K_DHEAD; + int cap_args = 0; + if (ab > aa) { + dir.args = (const char **)rst_grow(d, NULL, 0, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs] = rst_dup_stripped(d, L[i].s + aa, ab - aa); + dir.nargs++; + } + int j = i + 1; + int ka; + int kb; + int va; + int vb; + while (j < n && !L[j].blank && L[j].indent > ind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + dir.args = (const char **)rst_grow(d, dir.args, dir.nargs, &cap_args, sizeof(char *)); + if (!dir.args) { + return n; + } + dir.args[dir.nargs++] = rst_dup_stripped(d, L[j].s, L[j].len); + d->K[j] = prose_arg ? K_DARG_PROSE : K_DARG; + j++; + } + int cap_opts = 0; + while (j < n && !L[j].blank && L[j].indent > ind && + rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *key = cbm_arena_strndup(d->scratch, L[j].s + ka, (size_t)(kb - ka)); + char *val = cbm_arena_strndup(d->scratch, L[j].s + va, (size_t)(vb - va)); + int oind = L[j].indent; + d->K[j] = K_DOPT; + j++; + while (j < n && !L[j].blank && L[j].indent > oind && + !rst_option_match(L[j].s, L[j].len, &ka, &kb, &va, &vb)) { + char *more = rst_dup_stripped(d, L[j].s, L[j].len); + if (val && more) { + val = cbm_arena_sprintf(d->scratch, "%s%s%s", val, val[0] && more[0] ? " " : "", + more); + } + d->K[j] = K_DOPT; + j++; + } + dir.opts = (rst_opt_t *)rst_grow(d, dir.opts, dir.nopts, &cap_opts, sizeof(rst_opt_t)); + if (!key || !val || !dir.opts) { + d->failed = true; + return n; + } + /* options[key] = val: a repeated key keeps its first place, the last value */ + bool found = false; + for (int k = 0; k < dir.nopts; k++) { + if (strcmp(dir.opts[k].key, key) == 0) { + dir.opts[k].val = val; + found = true; + } + } + if (!found) { + dir.opts[dir.nopts++] = (rst_opt_t){key, val}; + } + } + dir.cs = j; + dir.ce = rst_block_end(d, j, ind); + d->dirs = (rst_dir_t *)rst_grow(d, d->dirs, d->ndirs, &d->cap_dirs, sizeof(rst_dir_t)); + if (!d->dirs) { + return n; + } + d->dirs[d->ndirs++] = dir; + if (rst_in(dir.dname, LITERAL_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_LITERAL); + return dir.ce; + } + if (strcmp(dir.dname, "parsed-literal") == 0) { + rst_mark(d, dir.cs, dir.ce, K_PLIT); + return dir.ce; + } + if (rst_in(dir.dname, SKIP_CONTENT)) { + rst_mark(d, dir.cs, dir.ce, K_SKIP); + return dir.ce; + } + return dir.cs; +} + +static int rst_explicit(rst_doc_t *d, int i, int ind) { + rst_line_t *L = d->L; + int a; + int b; + rst_strip(L[i].s, L[i].len, &a, &b); + const char *s = L[i].s + a; + int sl = b - a; + if ((sl >= 4 && strncmp(s, ".. _", 4) == 0) || (sl >= 3 && strncmp(s, "__ ", 3) == 0)) { + d->K[i] = K_TARGET; + int j = i + 1; + while (j < d->n && !L[j].blank && L[j].indent > ind) { + d->K[j] = K_TARGET; + j++; + } + return j; + } + /* \.\.\s+\[[^\]]+\] : a footnote or citation */ + int p = rst_skip_ws(s, sl, 2); + if (p > 2 && p < sl && s[p] == '[') { + const char *close = memchr(s + p + 1, ']', (size_t)(sl - p - 1)); + if (close && close > s + p + 1) { + d->K[i] = K_PROSE; + return i + 1; + } + } + if (sl == 2 && (i + 1 >= d->n || L[i + 1].blank)) { + d->K[i] = K_COMMENT; + return i + 1; + } + int j = rst_block_end(d, i + 1, ind); + rst_mark(d, i, j, K_COMMENT); + return j; +} + +static void rst_scan(rst_doc_t *d) { + rst_line_t *L = d->L; + int n = d->n; + int i = 0; + int para_indent = 0; + int literal_pending = -1; + while (i < n && !d->failed) { + if (L[i].blank) { + d->K[i] = K_BLANK; + i++; + continue; + } + bool prev_blank = i == 0 || L[i - 1].blank; + int ind = L[i].indent; + if (literal_pending >= 0) { + int lp = literal_pending; + literal_pending = -1; + if (prev_blank && ind > lp) { + int j = rst_block_end(d, i, lp); + rst_mark(d, i, j, K_LITERAL); + i = j; + continue; + } + } + if (ind == 0 && prev_blank) { + int tidx; + char ch; + bool over; + int next; + if (rst_title_at(d, i, &tidx, &ch, &over, &next)) { + int level; + d->titles = (rst_title_t *)rst_grow(d, d->titles, d->ntitles, &d->cap_titles, + sizeof(rst_title_t)); + char *text = rst_collapse(d->scratch, L[tidx].s, L[tidx].len); + if (!d->titles || !text || !rst_style_index(d, ch, over, &level)) { + d->failed = true; + return; + } + (void)level; + d->titles[d->ntitles++] = + (rst_title_t){.first = i, .idx = tidx, .next = next, .text = text}; + rst_mark(d, i, next, K_ADORN); + d->K[tidx] = K_TITLE; + i = next; + continue; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (rst_adorn(L[i].s, L[i].len) && rst_cps(L[i].s, sa, sb) >= RST_SHORT_ADORN && + (i + 1 >= n || L[i + 1].blank)) { + d->K[i] = K_ADORN; + i++; + continue; + } + } + int dind; + int na; + int nb; + int aa; + int ab; + if (rst_directive_match(L[i].s, L[i].len, &dind, &na, &nb, &aa, &ab)) { + i = rst_directive(d, i, dind, na, nb, aa, ab); + continue; + } + if (rst_explicit_match(L[i].s, L[i].len)) { + i = rst_explicit(d, i, ind); + continue; + } + d->K[i] = K_PROSE; + if (prev_blank) { + int mm = rst_list_marker(L[i].s, L[i].len); + para_indent = mm >= 0 ? mm : ind; + } + int sa; + int sb; + rst_strip(L[i].s, L[i].len, &sa, &sb); + if (sb - sa >= 2 && L[i].s[sb - 1] == ':' && L[i].s[sb - 2] == ':') { + literal_pending = para_indent; + } + i++; + } +} + +/* ── Python signatures (Sphinx py_sig_re, H3 join_sig_lines / py_fullname) ── */ + +/* One signature per argument line; a trailing backslash continues it. */ +static int rst_join_sigs(rst_doc_t *d, const rst_dir_t *dir, const char ***out) { + const char **sigs = + (const char **)cbm_arena_alloc(d->scratch, (size_t)(dir->nargs + 1) * sizeof(char *)); + if (!sigs) { + d->failed = true; + return 0; + } + int ns = 0; + const char *cur = ""; + bool open = false; + for (int k = 0; k < dir->nargs; k++) { + const char *a = dir->args[k]; + size_t al = strlen(a); + if (al > 0 && a[al - 1] == '\\') { + cur = cbm_arena_sprintf(d->scratch, "%s%.*s ", cur, (int)(al - 1), a); + open = true; + } else { + sigs[ns++] = cbm_arena_sprintf(d->scratch, "%s%s", cur, a); + cur = ""; + open = false; + } + if (!cur || (ns > 0 && !sigs[ns - 1])) { + d->failed = true; + return 0; + } + } + if (open) { + sigs[ns++] = cur; + } + *out = sigs; + return ns; +} + +/* After the name: \s* (\[ .* \])? \s* ( \( .* \) (\s* -> .*)? )? $ */ +static bool rst_sig_paren_ok(const char *s, int len, int i) { + if (i == len) { + return true; + } + if (s[i] != '(') { + return false; + } + for (int q = len - 1; q > i; q--) { + if (s[q] != ')') { + continue; + } + if (q + 1 == len) { + return true; + } + int w = rst_skip_ws(s, len, q + 1); + if (w + RST_SIG_ARROW <= len && s[w] == '-' && s[w + 1] == '>') { + return true; + } + } + return false; +} + +static bool rst_sig_rest_ok(const char *s, int len, int i) { + i = rst_skip_ws(s, len, i); + if (i == len) { + return true; + } + if (s[i] == '[') { + for (int q = len - 1; q > i; q--) { + if (s[q] == ']' && rst_sig_paren_ok(s, len, rst_skip_ws(s, len, q + 1))) { + return true; + } + } + return false; + } + return rst_sig_paren_ok(s, len, i); +} + +/* PY_SIG_RE on sig.strip(): the prefix ("a.b." or "") and the name. */ +static bool rst_py_sig(rst_doc_t *d, const char *sig, const char **prefix, const char **name) { + int a; + int b; + rst_strip(sig, (int)strlen(sig), &a, &b); + const char *s = sig + a; + int len = b > a ? b - a : 0; + int r = 0; + int last_dot = -1; + while (r < len) { + int n; + uint32_t cp = rst_cp(s, len, r, &n); + if (cp == '.') { + last_dot = r; + } else if (!rst_word_cp(cp)) { + break; + } + r += n; + } + int ns = last_dot + 1; + if (ns >= r || !rst_sig_rest_ok(s, len, r)) { + return false; + } + *prefix = cbm_arena_strndup(d->scratch, s, (size_t)ns); + *name = cbm_arena_strndup(d->scratch, s + ns, (size_t)(r - ns)); + if (!*prefix || !*name) { + d->failed = true; + return false; + } + return true; +} + +/* Sphinx PyObject.handle_signature's fullname, and the remaining prefix. */ +static void rst_py_fullname(rst_doc_t *d, const char *prefix, const char *name, const char *cls, + const char **full, const char **rem) { + if (cls && cls[0]) { + size_t cl = strlen(cls); + if (prefix[0] && + (strcmp(prefix, cls) == 0 || (strncmp(prefix, cls, cl) == 0 && prefix[cl] == '.'))) { + const char *r = prefix + cl; + while (*r == '.') { + r++; + } + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = r; + } else if (prefix[0]) { + *full = cbm_arena_sprintf(d->scratch, "%s.%s%s", cls, prefix, name); + *rem = prefix; + } else { + *full = cbm_arena_sprintf(d->scratch, "%s.%s", cls, name); + *rem = ""; + } + } else { + *full = cbm_arena_sprintf(d->scratch, "%s%s", prefix, name); + *rem = prefix; + } + if (!*full) { + d->failed = true; + *full = ""; + } +} + +static const char *rst_opt(const rst_dir_t *dir, const char *key) { + for (int k = 0; k < dir->nopts; k++) { + if (strcmp(dir->opts[k].key, key) == 0) { + return dir->opts[k].val; + } + } + return NULL; +} + +/* H3 _contexts: the module, C namespace and class in force on every line. */ +static void rst_contexts(rst_doc_t *d) { + int n = d->n; + d->mod_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->ns_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + d->cls_at = (const char **)cbm_arena_alloc(d->scratch, (size_t)n * sizeof(char *)); + if (!d->mod_at || !d->ns_at || !d->cls_at) { + d->failed = true; + return; + } + memset(d->cls_at, 0, (size_t)n * sizeof(char *)); + const char *cm = NULL; + const char *cn = NULL; + int k = 0; + /* directives are in line order (the scan appends them so) */ + for (int di = 0; di <= d->ndirs; di++) { + int upto = di < d->ndirs ? d->dirs[di].line : n; + for (; k < upto; k++) { + d->mod_at[k] = cm; + d->ns_at[k] = cn; + } + if (di == d->ndirs) { + break; + } + const rst_dir_t *dir = &d->dirs[di]; + const char *v = dir->nargs ? dir->args[0] : NULL; + if (strcmp(dir->dom, "py") == 0 && + (strcmp(dir->dname, "module") == 0 || strcmp(dir->dname, "currentmodule") == 0)) { + cm = (!v || !v[0] || strcmp(v, "None") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && + (strcmp(dir->dname, "namespace") == 0 || strcmp(dir->dname, "namespace-push") == 0)) { + cn = (!v || !v[0] || strcmp(v, "NULL") == 0) ? NULL : v; + } + if (strcmp(dir->dom, "c") == 0 && strcmp(dir->dname, "namespace-pop") == 0) { + cn = NULL; + } + } + for (int di = 0; di < d->ndirs && !d->failed; di++) { + const rst_dir_t *dir = &d->dirs[di]; + if (strcmp(dir->dom, "py") != 0 || !rst_in(dir->dname, PY_OBJ) || + strcmp(dir->dname, "module") == 0 || dir->cs >= dir->ce) { + continue; + } + const char *outer = d->cls_at[dir->line]; + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + const char *content_cls = outer; + const char *prefix; + const char *name; + if (ns > 0 && rst_py_sig(d, sigs[ns - 1], &prefix, &name)) { + const char *full; + const char *rem; + rst_py_fullname(d, prefix, name, outer, &full, &rem); + if (strcmp(dir->dname, "class") == 0 || strcmp(dir->dname, "exception") == 0) { + content_cls = full; + } else if (rem[0]) { + size_t rl = strlen(rem); + const char *r = rem; + while (*r == '.') { + r++; + rl--; + } + while (rl > 0 && r[rl - 1] == '.') { + rl--; + } + content_cls = cbm_arena_strndup(d->scratch, r, rl); + } + } + const char *mod = rst_opt(dir, "module"); + if (mod) { + for (int j = dir->cs; j < dir->ce; j++) { + d->mod_at[j] = mod; + } + } + for (int j = dir->cs; j < dir->ce; j++) { + d->cls_at[j] = content_cls; + } + } +} + +/* ── Tokens ──────────────────────────────────────────────────────── */ + +static void rst_tok(rst_doc_t *d, int line, int col, int syntax, const char *raw) { + if (!raw) { + d->failed = true; + return; + } + d->toks = (rst_tok_t *)rst_grow(d, d->toks, d->ntoks, &d->cap_toks, sizeof(rst_tok_t)); + if (!d->toks) { + return; + } + d->toks[d->ntoks++] = (rst_tok_t){.line = line, .col = col, .syntax = syntax, .raw = raw}; +} + +static const char *rst_s(const char *s) { + return s ? s : ""; +} + +/* The written form of a directive: its head line, whitespace collapsed. */ +static const char *rst_written_dir(rst_doc_t *d, const rst_dir_t *dir) { + const rst_line_t *ln = &d->L[dir->line]; + return rst_collapse(d->ctx->arena, ln->s, ln->len); +} + +static bool rst_code_ext(const char *path) { + const char *base = strrchr(path, '/'); + base = base ? base + 1 : path; + if (strncmp(base, "Makefile", strlen("Makefile")) == 0 || + strncmp(base, "Kconfig", strlen("Kconfig")) == 0) { + return true; + } + const char *dot = strrchr(base, '.'); + if (!dot) { + return false; + } + char ext[16]; + size_t el = strlen(dot); + if (el >= sizeof(ext)) { + return false; + } + for (size_t i = 0; i <= el; i++) { + char c = dot[i]; + ext[i] = (char)(c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c); + } + return rst_in(ext, CODE_EXT); +} + +static void rst_directive_tokens(rst_doc_t *d, const rst_dir_t *dir) { + CBMArena *a = d->ctx->arena; + const char *dn = dir->dname; + const char *dom = dir->dom; + bool py_obj = (strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(dn, PY_OBJ); + bool c_obj = (strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0 || strcmp(dom, "py") == 0) && + rst_in(dn, C_OBJ); + if (strcmp(dom, "py") != 0 && strcmp(dom, "c") != 0 && strcmp(dom, "std") != 0) { + py_obj = false; + c_obj = false; + } + if (strcmp(dom, "c") == 0) { + py_obj = false; + } + int line = dir->line; + const char *written = NULL; + if (py_obj || c_obj) { + written = rst_written_dir(d, dir); + const char *mod = rst_opt(dir, "module"); + const char **sigs = NULL; + int ns = rst_join_sigs(d, dir, &sigs); + if (ns == 0 && !d->failed) { + sigs = (const char **)cbm_arena_alloc(d->scratch, sizeof(char *)); + if (!sigs) { + d->failed = true; + return; + } + sigs[0] = ""; + ns = 1; + } + bool attr_like = + strcmp(dn, "attribute") == 0 || strcmp(dn, "data") == 0 || strcmp(dn, "property") == 0; + for (int k = 0; k < ns && !d->failed; k++) { + char *sig = rst_collapse(d->scratch, sigs[k], (int)strlen(sigs[k])); + if (!sig) { + d->failed = true; + return; + } + /* Python's reading (H3 py_directive): an attribute's type or + * value is no part of its name; the full name in the class */ + char *py_sig = sig; + if (attr_like) { + size_t cut = strcspn(sig, ":="); + while (cut > 0 && sig[cut - 1] == ' ') { + cut--; + } + py_sig = cbm_arena_strndup(d->scratch, sig, cut); + } + const char *prefix; + const char *name; + const char *full = ""; + const char *rem; + if (py_sig && strcmp(dn, "module") != 0 && rst_py_sig(d, py_sig, &prefix, &name)) { + rst_py_fullname(d, prefix, name, d->cls_at[line], &full, &rem); + } + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_OBJECT, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s\t%s", rst_s(written), dir->name, + sig, rst_s(mod ? mod : d->mod_at[line]), + rst_s(d->cls_at[line]), rst_s(d->ns_at[line]), full)); + } + return; + } + const char *arg0 = dir->nargs ? dir->args[0] : ""; + if (rst_in(dn, AUTODOC)) { + written = rst_written_dir(d, dir); + char *arg = rst_collapse(d->scratch, arg0, (int)strlen(arg0)); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_AUTODOC, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), dn, rst_s(arg), + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]))); + return; + } + if (strcmp(dn, "literalinclude") == 0) { + written = rst_written_dir(d, dir); + const char *lines = rst_opt(dir, "lines"); + const char *pyobject = rst_opt(dir, "pyobject"); + bool bounded = rst_opt(dir, "start-after") || rst_opt(dir, "end-before") || + rst_opt(dir, "start-at") || rst_opt(dir, "end-at"); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_LITERALINCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s", rst_s(written), arg0, rst_s(lines), + rst_s(pyobject), bounded ? "1" : "")); + return; + } + if (strcmp(dn, "include") == 0 || strcmp(dn, "kernel-include") == 0) { + if (!arg0[0] || arg0[0] == '<' || !rst_code_ext(arg0)) { + return; /* a document or text include: not code */ + } + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_INCLUDE, + cbm_arena_sprintf(a, "%s\t%s\t%s", rst_s(written), dn, arg0)); + return; + } + if (strcmp(dn, "kernel-doc") == 0) { + written = rst_written_dir(d, dir); + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t", rst_s(written), arg0)); + const char *ids = rst_opt(dir, "identifiers"); + if (!ids) { + ids = rst_opt(dir, "functions"); + } + if (!ids || rst_opt(dir, "export") || rst_opt(dir, "internal") || rst_opt(dir, "doc")) { + return; /* the names come from the source file's own kernel-doc comments */ + } + /* each name, a trailing backslash dropped (kerneldoc.py) */ + const char *p = ids; + while (*p && !d->failed) { + while (*p == ' ') { + p++; + } + const char *e = p; + while (*e && *e != ' ') { + e++; + } + size_t nl = (size_t)(e - p); + while (nl > 0 && p[nl - 1] == '\\') { + nl--; + } + if (nl > 0) { + rst_tok(d, line, dir->indent, CBM_DOCLINK_RST_KERNEL_DOC, + cbm_arena_sprintf(a, "%s\t%s\t%.*s", rst_s(written), arg0, (int)nl, p)); + } + p = e; + } + } +} + +/* ── Inline text: roles and inline literals ──────────────────────── */ + +static bool rst_inline_kind(unsigned char k) { + return k == K_PROSE || k == K_TITLE || k == K_DHEAD_PROSE || k == K_DARG_PROSE || k == K_PLIT; +} + +typedef struct { + char *text; /* the run's lines joined by '\n' */ + char *mask; /* inline literals blanked (H3 mask) */ + int len; + int a; /* first line */ + int *offs; + int nlines; +} rst_run_t; + +static int rst_run_line(const rst_run_t *r, int pos) { + int lo = 0; + int hi = r->nlines; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (r->offs[mid] <= pos) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo - 1; +} + +/* Is the code point before byte i a role's left neighbour that blocks it + * ((? 0 && ((unsigned char)s[k] & 0xC0) == 0x80) { + k--; + } + int n; + uint32_t cp = rst_cp(s, i, k, &n); + return cp == '`' || cp == '\\' || rst_word_cp(cp); +} + +/* [A-Za-z][\w.+-]* at s[i]: its end, or -1. */ +static int rst_role_name(const char *s, int len, int i) { + if (i >= len || !rst_alpha_ascii((unsigned char)s[i])) { + return -1; + } + i++; + while (i < len) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (cp != '.' && cp != '+' && cp != '-' && !rst_word_cp(cp)) { + break; + } + i += n; + } + return i; +} + +static bool rst_role_end_ok(const char *s, int len, int i) { + if (i >= len) { + return true; + } + int n; + uint32_t cp = rst_cp(s, len, i, &n); + return cp != '`' && !rst_word_cp(cp); +} + +static void rst_emit_role(rst_doc_t *d, const rst_run_t *r, int start, int role_a, int role_b, + int ca, int cb) { + CBMArena *a = d->ctx->arena; + char *role = cbm_arena_strndup(d->scratch, r->text + role_a, (size_t)(role_b - role_a)); + if (!role) { + d->failed = true; + return; + } + const char *dom; + const char *rname; + rst_split_domain(d, role, PY_ROLES, &dom, &rname); + bool code = ((strcmp(dom, "py") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, PY_ROLES)) || + ((strcmp(dom, "c") == 0 || strcmp(dom, "std") == 0) && rst_in(rname, C_ROLES)); + char *content = rst_collapse(d->scratch, r->text + ca, cb - ca); + if (!content) { + d->failed = true; + return; + } + if (!code) { + /* an extlink to a repository path (:source:`django/db/x.py`, or with a + * title, :source:`x `): only a role without a domain + * whose target is a path */ + const char *target = content; + size_t tl = strlen(content); + const char *lt = strrchr(content, '<'); + if (tl > 0 && content[tl - 1] == '>' && lt && lt > content) { + target = lt + 1; + tl = (size_t)(content + tl - 1 - target); + } + CBMDocLinkMdPath p; + if (strchr(role, ':') || + !cbm_doclink_md_classify_span(target, tl, d->span_buf, RST_SPAN_MAX + 1, &p) || + p.shape == CBM_DOCLINK_MD_QUALIFIED) { + return; + } + } + int li = rst_run_line(r, start); + int line = r->a + li; + char *written = rst_collapse(a, r->text + start, cb + 1 - start); + rst_tok(d, line, start - r->offs[li], CBM_DOCLINK_RST_ROLE, + written ? cbm_arena_sprintf(a, "%s\t%s\t%s\t%s\t%s\t%s", written, role, content, + rst_s(d->mod_at[line]), rst_s(d->cls_at[line]), + rst_s(d->ns_at[line])) + : NULL); +} + +/* ROLE_RE over the masked text. */ +static void rst_roles(rst_doc_t *d, const rst_run_t *r) { + const char *m = r->mask; + int len = r->len; + int i = 0; + while (i < len && !d->failed) { + const char *colon = memchr(m + i, ':', (size_t)(len - i)); + if (!colon) { + break; + } + i = (int)(colon - m); + if (rst_role_blocked(m, i)) { + i++; + continue; + } + int e1 = rst_role_name(m, len, i + 1); + int role_end = -1; + if (e1 > 0 && e1 < len && m[e1] == ':') { + int e2 = rst_role_name(m, len, e1 + 1); + if (e2 > 0 && e2 + 1 < len && m[e2] == ':' && m[e2 + 1] == '`') { + role_end = e2; + } else if (e1 + 1 < len && m[e1 + 1] == '`') { + role_end = e1; + } + } + if (role_end < 0) { + i++; + continue; + } + int open = role_end + 1; + const char *close = memchr(m + open + 1, '`', (size_t)(len - open - 1)); + if (!close || close == m + open + 1 || !rst_role_end_ok(m, len, (int)(close - m) + 1)) { + i++; + continue; + } + int cb = (int)(close - m); + rst_emit_role(d, r, i, i + 1, role_end, open + 1, cb); + i = cb + 1; + } +} + +static void rst_literal_token(rst_doc_t *d, const rst_run_t *r, int i, int a, int b) { + if (b <= a || b - a > RST_SPAN_MAX || memchr(r->text + a, '\n', (size_t)(b - a))) { + return; + } + CBMDocLinkMdPath p; + if (!cbm_doclink_md_classify_span(r->text + a, (size_t)(b - a), d->span_buf, RST_SPAN_MAX + 1, + &p)) { + return; + } + int li = rst_run_line(r, i); + rst_tok(d, r->a + li, i - r->offs[li], + p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_RST_CODE_NAME + : CBM_DOCLINK_RST_CODE_PATH, + cbm_arena_strndup(d->ctx->arena, r->text + a, (size_t)(b - a))); +} + +/* INLINE_LIT_RE ``.+?`` (DOTALL): blank them in the mask, and take each as a + * possible code_path / code_name. */ +static void rst_literals(rst_doc_t *d, rst_run_t *r) { + const char *t = r->text; + int len = r->len; + int i = 0; + while (i + 1 < len && !d->failed) { + if (t[i] != '`' || t[i + 1] != '`') { + i++; + continue; + } + int j = i + 3; + while (j + 1 < len && !(t[j] == '`' && t[j + 1] == '`')) { + j++; + } + if (j + 1 >= len) { + i++; + continue; + } + for (int k = i; k < j + 2; k++) { + if (r->mask[k] != '\n') { + r->mask[k] = ' '; + } + } + rst_literal_token(d, r, i, i + 2, j); + i = j + 2; + } +} + +static void rst_inline(rst_doc_t *d) { + int i = 0; + while (i < d->n && !d->failed) { + if (!rst_inline_kind(d->K[i])) { + i++; + continue; + } + int j = i; + int len = 0; + while (j < d->n && rst_inline_kind(d->K[j])) { + len += d->L[j].len + 1; + j++; + } + rst_run_t r = {.a = i, .nlines = j - i}; + r.text = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.mask = (char *)cbm_arena_alloc(d->scratch, (size_t)len + 1); + r.offs = (int *)cbm_arena_alloc(d->scratch, (size_t)(j - i) * sizeof(int)); + if (!r.text || !r.mask || !r.offs) { + d->failed = true; + return; + } + int pos = 0; + for (int k = i; k < j; k++) { + r.offs[k - i] = pos; + memcpy(r.text + pos, d->L[k].s, (size_t)d->L[k].len); + pos += d->L[k].len; + if (k + 1 < j) { + r.text[pos++] = '\n'; + } + } + r.text[pos] = '\0'; + r.len = pos; + memcpy(r.mask, r.text, (size_t)pos + 1); + rst_literals(d, &r); + rst_roles(d, &r); + i = j; + } +} + +/* ── Sections ────────────────────────────────────────────────────── */ + +/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */ +static const char *rst_qn_segment(CBMArena *a, const char *name) { + char *out = cbm_arena_strdup(a, name); + if (!out) { + return NULL; + } + size_t w = 0; + bool in_ws = false; + for (const char *p = name; *p; p++) { + if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') { + in_ws = true; + continue; + } + if (in_ws && w > 0) { + out[w++] = '-'; + } + in_ws = false; + out[w++] = *p; + } + out[w] = '\0'; + return out; +} + +/* The section's own text (title block to the next title), whitespace + * collapsed, at most RST_BODY_MAX bytes and never a cut character. */ +static const char *rst_body(rst_doc_t *d, int from, int to) { + char *out = (char *)cbm_arena_alloc(d->ctx->arena, RST_BODY_MAX + 1); + if (!out) { + return NULL; + } + int w = 0; + for (int k = from; k < to && w < RST_BODY_MAX; k++) { + const char *s = d->L[k].s; + int len = d->L[k].len; + int i = 0; + while (i < len && w < RST_BODY_MAX) { + int n; + uint32_t cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + i += n; + continue; + } + if (w > 0) { + out[w++] = ' '; + } + while (i < len) { + cp = rst_cp(s, len, i, &n); + if (rst_space_cp(cp)) { + break; + } + if (w + n > RST_BODY_MAX) { + w = RST_BODY_MAX + 1; /* stop: the next character does not fit */ + break; + } + memcpy(out + w, s + i, (size_t)n); + w += n; + i += n; + } + } + } + if (w > RST_BODY_MAX) { + w = RST_BODY_MAX; + while (w > 0 && ((unsigned char)out[w] & 0xC0) == 0x80) { + w--; + } + } + while (w > 0 && out[w - 1] == ' ') { + w--; + } + out[w] = '\0'; + return w > 0 ? out : NULL; +} + +static int rst_title_qn_cmp(const void *a, const void *b) { + const rst_title_t *x = *(const rst_title_t *const *)a; + const rst_title_t *y = *(const rst_title_t *const *)b; + return strcmp(x->qn, y->qn); +} + +static void rst_sections(rst_doc_t *d) { + CBMExtractCtx *ctx = d->ctx; + CBMArena *a = ctx->arena; + for (int t = 0; t < d->ntitles; t++) { + rst_title_t *ti = &d->titles[t]; + const char *seg = rst_qn_segment(a, ti->text); + ti->qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL; + if (!ti->qn) { + d->failed = true; + return; + } + } + if (d->ntitles > 1) { + rst_title_t **by_qn = + (rst_title_t **)cbm_arena_alloc(d->scratch, (size_t)d->ntitles * sizeof(*by_qn)); + if (!by_qn) { + d->failed = true; + return; + } + for (int t = 0; t < d->ntitles; t++) { + by_qn[t] = &d->titles[t]; + } + qsort(by_qn, (size_t)d->ntitles, sizeof(*by_qn), rst_title_qn_cmp); + for (int t = 1; t < d->ntitles; t++) { + if (strcmp(by_qn[t]->qn, by_qn[t - 1]->qn) == 0) { + by_qn[t]->shared_qn = true; + by_qn[t - 1]->shared_qn = true; + } + } + } + for (int t = 0; t < d->ntitles; t++) { + const rst_title_t *ti = &d->titles[t]; + int end = t + 1 < d->ntitles ? d->titles[t + 1].first : d->n; + int last = end - 1; + while (last > ti->idx && d->L[last].blank) { + last--; + } + char *name = cbm_arena_strdup(a, ti->text); + if (!name) { + d->failed = true; + return; + } + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = name; + def.qualified_name = ti->qn; + def.label = "Section"; + def.file_path = ctx->rel_path; + def.start_line = (uint32_t)ti->idx + 1; + def.end_line = (uint32_t)last + 1; + def.is_exported = true; + def.docstring = rst_body(d, ti->next, end); + cbm_defs_push(&ctx->result->defs, a, def); + } +} + +/* ── Driver ──────────────────────────────────────────────────────── */ + +static int rst_tok_cmp(const void *a, const void *b) { + const rst_tok_t *x = (const rst_tok_t *)a; + const rst_tok_t *y = (const rst_tok_t *)b; + if (x->line != y->line) { + return x->line < y->line ? -1 : 1; + } + if (x->col != y->col) { + return x->col < y->col ? -1 : 1; + } + if (x->syntax != y->syntax) { + return x->syntax < y->syntax ? -1 : 1; + } + return strcmp(x->raw, y->raw); +} + +/* The section a 0-based line is written in (by its title block's first line). */ +static const rst_title_t *rst_section_of(const rst_doc_t *d, int line) { + int lo = 0; + int hi = d->ntitles; + while (lo < hi) { + int mid = lo + ((hi - lo) / 2); + if (d->titles[mid].first <= line) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo > 0 ? &d->titles[lo - 1] : NULL; +} + +void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx) { + if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) { + return; + } + rst_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena}; + d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(RST_SPAN_MAX + 1) * 2); + if (!d.span_buf || !rst_lines(&d, ctx->source, ctx->source_len)) { + ctx->result->doc_links.failed = true; + return; + } + rst_scan(&d); + if (!d.failed) { + rst_contexts(&d); + } + if (!d.failed) { + rst_sections(&d); + } + for (int i = 0; i < d.ndirs && !d.failed; i++) { + rst_directive_tokens(&d, &d.dirs[i]); + } + if (!d.failed) { + rst_inline(&d); + } + if (d.failed) { + ctx->result->doc_links.failed = true; + return; + } + qsort(d.toks, (size_t)d.ntoks, sizeof(*d.toks), rst_tok_cmp); + for (int i = 0; i < d.ntoks; i++) { + const rst_tok_t *t = &d.toks[i]; + const rst_title_t *sec = rst_section_of(&d, t->line); + if (sec && sec->shared_qn) { + sec = NULL; /* the file, not the node another title owns */ + } + CBMDocLink link = { + .source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""), + .raw = t->raw, + .line = (uint32_t)t->line + 1, + .def_line = sec ? (uint32_t)sec->idx + 1 : 1, + .syntax = (uint16_t)t->syntax, + .flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE, + }; + cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link); + } + /* an architecture decision record in reST: its node, facts and supersedes */ + cbm_adr_extract(ctx); +} diff --git a/src/discover/discover.c b/src/discover/discover.c index d2b02e217..26a5ab4d4 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -16,6 +16,7 @@ #include "foundation/constants.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/mem_core.h" #include "foundation/workspace.h" #include "foundation/platform.h" #ifdef _WIN32 @@ -888,13 +889,159 @@ static int safe_stat(const char *abs_path, struct stat *st, bool *is_symlink) { #endif } +/* ── Sphinx documentation sets ──────────────────────────────────── + * A directory with a conf.py is a Sphinx documentation set, and its + * `source_suffix` says which file suffixes are reStructuredText (Django's + * docs are `.txt`). A file of such a suffix in the set's subtree, of no + * language by its name, is reST; a nested conf.py starts its own set. conf.py + * is read as text (the first SPHINX_CONF_READ bytes) and never run. */ + +enum { + SPHINX_SUFFIXES = 4, /* extra reST suffixes one set declares */ + SPHINX_SUFFIX_LEN = 16, /* ".txt", ".rest": a longer one is no suffix */ + SPHINX_CONF_READ = 65536, +}; + +typedef struct { + char ext[SPHINX_SUFFIXES][SPHINX_SUFFIX_LEN]; + int count; +} sphinx_suffixes_t; + +/* The quoted string at s[i] ('...' or "..."): its span; false otherwise. */ +static bool sphinx_quoted(const char *s, size_t n, size_t i, size_t *a, size_t *b) { + if (i >= n || (s[i] != '\'' && s[i] != '"')) { + return false; + } + const char *close = memchr(s + i + SKIP_ONE, s[i], n - i - SKIP_ONE); + if (!close) { + return false; + } + *a = i + SKIP_ONE; + *b = (size_t)(close - s); + return true; +} + +static void sphinx_add(sphinx_suffixes_t *out, const char *s, size_t n) { + if (n < PAIR_LEN || n >= SPHINX_SUFFIX_LEN || s[0] != '.' || out->count >= SPHINX_SUFFIXES || + (n == strlen(".rst") && memcmp(s, ".rst", n) == 0)) { + return; + } + for (size_t i = SKIP_ONE; i < n; i++) { + if (!isalnum((unsigned char)s[i]) && s[i] != '_' && s[i] != '-') { + return; + } + } + memcpy(out->ext[out->count], s, n); + out->ext[out->count][n] = '\0'; + out->count++; +} + +/* `source_suffix = '.txt'`, `= ['.rst', '.txt']` or + * `= {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}` (a dict + * entry counts only when it maps to restructuredtext). */ +static void sphinx_parse(const char *s, size_t n, sphinx_suffixes_t *out) { + static const char key[] = "source_suffix"; + size_t kl = strlen(key); + for (size_t line = 0; line < n;) { + const char *nl = memchr(s + line, '\n', n - line); + size_t end = nl ? (size_t)(nl - s) : n; + size_t i = line + kl; + if (end >= i && memcmp(s + line, key, kl) == 0) { + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + if (i < n && s[i] == '=') { + i++; + while (i < n && (s[i] == ' ' || s[i] == '\t')) { + i++; + } + char open = i < n ? s[i] : '\0'; + char close = '\0'; + if (open == '[' || open == '(' || open == '{') { + close = open == '[' ? ']' : (open == '(' ? ')' : '}'); + } + /* the value: up to its closing bracket, or this line */ + size_t stop = end; + if (close) { + const char *c = memchr(s + i, close, n - i); + stop = c ? (size_t)(c - s) : end; + } + size_t k = i; + while (k < stop) { + size_t a; + size_t b; + if (!sphinx_quoted(s, stop, k, &a, &b)) { + k++; + continue; + } + k = b + SKIP_ONE; + if (open != '{') { + sphinx_add(out, s + a, b - a); + continue; + } + while (k < stop && (s[k] == ' ' || s[k] == ':')) { + k++; + } + size_t va; + size_t vb; + if (sphinx_quoted(s, stop, k, &va, &vb)) { + if (vb - va == strlen("restructuredtext") && + memcmp(s + va, "restructuredtext", vb - va) == 0) { + sphinx_add(out, s + a, b - a); + } + k = vb + SKIP_ONE; + } + } + return; /* the first assignment */ + } + } + line = end + SKIP_ONE; + } +} + +/* The documentation set this directory starts, if it has a conf.py. */ +static bool sphinx_load(const char *dir, sphinx_suffixes_t *out) { + char conf[CBM_SZ_4K]; + if (snprintf(conf, sizeof(conf), "%s/conf.py", dir) >= (int)sizeof(conf)) { + return false; + } + struct stat st; + if (wide_stat(conf, &st) != 0 || !S_ISREG(st.st_mode)) { + return false; + } + FILE *f = cbm_fopen(conf, "rb"); + if (!f) { + return false; + } + char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, SPHINX_CONF_READ); + size_t n = buf ? fread(buf, SKIP_ONE, SPHINX_CONF_READ, f) : 0; + (void)fclose(f); + memset(out, 0, sizeof(*out)); + if (buf) { + sphinx_parse(buf, n, out); + } + cbm_free(CBM_MEM_CLASS_OTHER, buf); + return true; +} + +static bool sphinx_is_rst(const sphinx_suffixes_t *set, const char *name) { + size_t nl = strlen(name); + for (int i = 0; set && i < set->count; i++) { + size_t el = strlen(set->ext[i]); + if (nl > el && strcmp(name + nl - el, set->ext[i]) == 0) { + return true; + } + } + return false; +} + /* Process a single regular file entry during directory walk. */ static void walk_dir_process_file(const char *abs_path, const char *rel_path, const char *name, const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, const struct stat *st, - file_list_t *out) { + const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx, + const struct stat *st, file_list_t *out) { const char *skip_reason = file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); if (skip_reason) { @@ -906,6 +1053,9 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co return; } CBMLanguage lang = detect_file_language(name, abs_path); + if (lang == CBM_LANG_COUNT && sphinx_is_rst(sphinx, name)) { + lang = CBM_LANG_RST; /* its documentation set's source_suffix */ + } if (lang == CBM_LANG_COUNT) { return; } @@ -916,6 +1066,7 @@ typedef struct { char dir[CBM_SZ_4K]; char prefix[CBM_SZ_4K]; const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */ + sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */ } walk_frame_t; /* Initial capacity only — the stack grows on demand. A single directory can * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855 @@ -973,6 +1124,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char return; } slot->ignore_chain = parent->ignore_chain; + slot->sphinx = parent->sphinx; ws->top++; } @@ -1021,7 +1173,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, &st, out); + cbmignore, &frame->sphinx, &st, out); } } @@ -1115,6 +1267,11 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis frame.ignore_chain = link; } + sphinx_suffixes_t set; + if (sphinx_load(frame.dir, &set)) { + frame.sphinx = set; /* a (nested) documentation set starts here */ + } + cbm_dir_t *d = cbm_opendir(frame.dir); if (!d) { if (out->count_only) { diff --git a/src/pipeline/doc_links.c b/src/pipeline/doc_links.c index 035fbf79a..f3b8f228d 100644 --- a/src/pipeline/doc_links.c +++ b/src/pipeline/doc_links.c @@ -48,6 +48,7 @@ static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = { &cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver, + &cbm_doclink_rst_resolver, }; enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) }; diff --git a/src/pipeline/doc_links.h b/src/pipeline/doc_links.h index ef09d15e6..8ad2384d4 100644 --- a/src/pipeline/doc_links.h +++ b/src/pipeline/doc_links.h @@ -242,6 +242,18 @@ extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver; extern const cbm_doclink_resolver_t cbm_doclink_md_resolver; /* PDF documents (doc_links_pdf.c): via "pdf". */ extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver; +/* reStructuredText documents and the Python scope they read (doc_links_rst.c): + * via "rst". */ +extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver; + +/* Over a Markdown resolver index (cbm_doclink_md_resolver.build), for the + * other document resolvers: the innermost code definition of `path` holding + * lines [first, last] (or the one named `member`), NULL for none; the Folder + * node at `path`; whether `path` is test code (a test directory or file). */ +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member); +const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path); +bool cbm_doclink_md_test_path(const char *path); #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS /* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 07db1d4f3..b2a76594e 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -1198,6 +1198,21 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD mdr_resolve_path(x, graph, doc, &r, out); } +/* ── Shared with the other document resolvers (doc_links.h) ──────── */ + +const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, + uint32_t first, uint32_t last, const char *member) { + return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; +} + +const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path) { + return md_index && path ? mdr_folder((const mdr_index_t *)md_index, NULL, path) : NULL; +} + +bool cbm_doclink_md_test_path(const char *path) { + return path && mdr_test_path(path); +} + const cbm_doclink_resolver_t cbm_doclink_md_resolver = { .langs = {CBM_LANG_MARKDOWN}, .lang_count = 1, diff --git a/src/pipeline/doc_links_rst.c b/src/pipeline/doc_links_rst.c new file mode 100644 index 000000000..f0c5377f5 --- /dev/null +++ b/src/pipeline/doc_links_rst.c @@ -0,0 +1,1713 @@ +/* + * doc_links_rst.c — reStructuredText references -> MENTIONS edges, via "rst" + * (the resolving half of internal/cbm/doclink_rst.c). + * + * Python domain (roles, object directives, autodoc) as the field test's + * resolver (H3 PyResolver) reads it. A dotted name is read like Python's + * import system: its longest prefix that is a module's import name (a .py + * file, or a package's __init__.py, named from its topmost package + * directory down) and the rest as a qualified name in that module; a + * package's re-export (`from .base import Model` in its __init__.py, read + * from the Python scope blobs of doclink_py.c) continues the lookup at its + * source. Forms in Sphinx's search order: T, Class.T, module.T, + * module.Class.T (a leading `.` searches the most specific first). EXACT + * when one form names a node. Otherwise UNIQUE by name: the one Python + * definition named like T's last piece whose qualified name ends with T -- + * for a T of two or more pieces only (a bare name is never searched: the + * field test's errors were bare names), under a module context the one + * inside the module when several are, and for an object directive (which + * declares its object IN its module) only inside it. + * + * C domain: a C identifier is the whole name -- the one C or C++ definition + * of that name and kind (H3 CResolver), a member through its struct. + * + * Paths: include / literalinclude (relative to the document; `/x` relative + * to its documentation set: the directory of the nearest conf.py), + * kernel-include and kernel-doc (relative to the repository root, as the + * kernel's Sphinx extensions read them), extlink roles whose URL is a + * repository blob path. A literalinclude's :lines: bind the innermost + * definition holding them, its :pyobject: the named definition. Inline + * literals: the Markdown resolver's code-span rules (doc_links_md.c). + * + * Unresolved: ambiguous (several), missing (the name's first piece or the + * path's directory is the repository's), test_only_target (only test code has + * it), unparseable. Anything else is LOCAL: another project's name (the + * standard library, an intersphinx name), a bare name no context resolves. + */ +#include "pipeline/doc_links.h" + +#include "doclink.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" +#include "graph_buffer/graph_buffer.h" + +#include +#include +#include +#include + +enum { + RR_PATH_CAP = 1024, /* the longest path or dotted name looked up */ + RR_RAW_CAP = 4096, /* the longest token record read */ + RR_FIELDS = 8, /* fields of a token record */ + RR_PIECES = 64, /* pieces of a dotted name */ + RR_ALIAS_DEPTH = 12, /* re-exports followed (H3) */ + RR_TABLE_INIT = 1024, +}; + +/* ── Index ───────────────────────────────────────────────────────── */ + +/* A Python module by its import name. */ +typedef struct rr_mod { + const char *file; /* its .py path */ + const char *qn_base; /* the QN prefix of its definitions: project + dotted path */ + bool is_pkg; /* an __init__.py */ + struct rr_mod *next; /* another module of the same import name */ +} rr_mod_t; + +/* A package's re-export: `from src import orig as alias` (src absolute). */ +typedef struct rr_alias { + const char *alias; + int level; + const char *src; /* as written (relative when level > 0) */ + const char *orig; + struct rr_alias *next; +} rr_alias_t; + +/* A Sphinx documentation set: the directory of its conf.py. */ +typedef struct { + const char *dir; + const char *primary; /* primary_domain ("py" when unset) */ + const char *const *ext; /* extlink roles to repository paths */ + int next_count; + const char *const *isx; /* intersphinx names */ + int nisx; +} rr_conf_t; + +typedef struct { + const char *project; + size_t project_len; + void *md; /* the Markdown resolver's index: code spans, segments, folders */ + const char **run_paths; + int run_count; + CBMArena arena; + CBMHashTable *mods; /* import name -> rr_mod_t list */ + CBMHashTable *roots; /* first piece of every import name */ + CBMHashTable *aliases; /* __init__.py path -> rr_alias_t list (first per alias) */ + CBMHashTable *modnode; /* .py path -> its Module node */ + rr_conf_t *confs; /* sorted by dir */ + int nconfs; +} rr_index_t; + +static void rr_destroy(void *index); + +static bool rr_ends(const char *s, const char *suffix) { + size_t n = strlen(s); + size_t k = strlen(suffix); + return n >= k && strcmp(s + n - k, suffix) == 0; +} + +static const char *rr_base(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +/* A python source file and its import name (from its topmost package + * directory: the directories above it holding an __init__.py). */ +static bool rr_add_module(rr_index_t *x, const CBMHashTable *pkgdirs, const char *path) { + size_t n = strlen(path); + if (n < strlen(".py") + 1 || n >= RR_PATH_CAP) { + return true; + } + char dotted[RR_PATH_CAP]; + size_t stem = n - strlen(".py"); + memcpy(dotted, path, stem); + dotted[stem] = '\0'; + bool is_pkg = strcmp(rr_base(path), "__init__.py") == 0; + if (is_pkg) { + char *slash = strrchr(dotted, '/'); + if (!slash) { + return true; /* a top-level __init__.py has no package name */ + } + *slash = '\0'; + } + /* the topmost package directory: walk up while the parent is a package */ + char dir[RR_PATH_CAP]; + snprintf(dir, sizeof(dir), "%s", dotted); + char *cut = is_pkg ? dir + strlen(dir) : strrchr(dir, '/'); + size_t start = 0; + if (!is_pkg && !cut) { + start = 0; /* a top-level module */ + } else { + if (cut) { + *cut = '\0'; + } + for (;;) { + if (!cbm_ht_get(pkgdirs, dir)) { + start = strlen(dir) + (dir[0] ? SKIP_ONE : 0); + break; + } + char *up = strrchr(dir, '/'); + if (!up) { + start = 0; + break; + } + *up = '\0'; + } + } + if (start >= strlen(dotted)) { + return true; + } + char *name = cbm_arena_strdup(&x->arena, dotted + start); + char *qn = cbm_arena_sprintf(&x->arena, "%s.%s", x->project, dotted); + rr_mod_t *m = (rr_mod_t *)cbm_arena_alloc(&x->arena, sizeof(*m)); + if (!name || !qn || !m) { + return false; + } + for (char *c = name; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + for (char *c = qn + x->project_len + SKIP_ONE; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + m->file = cbm_arena_strdup(&x->arena, path); + m->qn_base = qn; + m->is_pkg = is_pkg; + m->next = (rr_mod_t *)cbm_ht_get(x->mods, name); + if (!m->file) { + return false; + } + cbm_ht_set(x->mods, name, m); + size_t root_len = strcspn(name, "."); + char *root = cbm_arena_strndup(&x->arena, name, root_len); + if (!root) { + return false; + } + cbm_ht_set(x->roots, root, root); + return true; +} + +static bool rr_build_modules(rr_index_t *x, const cbm_gbuf_t *graph) { + const cbm_gbuf_node_t **files = NULL; + int count = 0; + if (cbm_gbuf_find_by_label(graph, "File", &files, &count) != 0) { + return true; + } + CBMHashTable *pkgdirs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + if (!pkgdirs) { + return false; + } + bool ok = true; + for (int i = 0; ok && i < count; i++) { + const char *fp = files[i]->file_path; + if (fp && strcmp(rr_base(fp), "__init__.py") == 0 && rr_base(fp) > fp) { + char *d = cbm_arena_strndup(&x->arena, fp, (size_t)(rr_base(fp) - fp - SKIP_ONE)); + ok = d != NULL; + if (ok) { + cbm_ht_set(pkgdirs, d, d); + } + } + } + for (int i = 0; ok && i < count; i++) { + const char *fp = files[i]->file_path; + if (fp && rr_ends(fp, ".py")) { + ok = rr_add_module(x, pkgdirs, fp); + } + } + cbm_ht_free(pkgdirs); + const cbm_gbuf_node_t **mods = NULL; + int nmods = 0; + if (ok && cbm_gbuf_find_by_label(graph, "Module", &mods, &nmods) == 0) { + for (int i = 0; i < nmods; i++) { + const char *fp = mods[i]->file_path; + if (fp && rr_ends(fp, ".py") && !cbm_ht_get(x->modnode, fp)) { + cbm_ht_set(x->modnode, fp, (void *)mods[i]); /* the graph outlives the index */ + } + } + } + return ok; +} + +/* Split a blob line into TAB-separated fields (in place). */ +static int rr_split(char *line, char **f, int cap) { + int n = 0; + char *p = line; + while (n < cap) { + f[n++] = p; + char *tab = strchr(p, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + p = tab + SKIP_ONE; + } + return n; +} + +static int rr_conf_cmp(const void *a, const void *b) { + return strcmp(((const rr_conf_t *)a)->dir, ((const rr_conf_t *)b)->dir); +} + +/* One Python scope blob (doclink_py.c): re-exports and conf.py settings. */ +static bool rr_read_scope(rr_index_t *x, const char *path, const char *scope, int *conf_cap) { + char *copy = cbm_arena_strdup(&x->arena, scope); + if (!copy) { + return false; + } + rr_conf_t conf = {0}; + bool is_conf = false; + const char **ext = NULL; + const char **isx = NULL; + int next = 0; + int nisx = 0; + char *save = NULL; + for (char *line = strtok_r(copy, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) { + char *f[RR_FIELDS]; + int nf = rr_split(line, f, RR_FIELDS); + if (strcmp(f[0], "I") == 0 && nf == 5) { + rr_alias_t *a = (rr_alias_t *)cbm_arena_alloc(&x->arena, sizeof(*a)); + if (!a) { + return false; + } + *a = (rr_alias_t){.alias = f[4], .level = atoi(f[1]), .src = f[2], .orig = f[3]}; + rr_alias_t *head = (rr_alias_t *)cbm_ht_get(x->aliases, path); + if (!head) { + cbm_ht_set(x->aliases, path, a); + } else { + rr_alias_t *t = head; + bool seen = false; + for (;; t = t->next) { + seen = seen || strcmp(t->alias, a->alias) == 0; + if (!t->next) { + break; + } + } + if (!seen) { + t->next = a; /* the first import of a name binds it (setdefault) */ + } + } + } else if (strcmp(f[0], "C") == 0) { + is_conf = true; + } else if (strcmp(f[0], "D") == 0 && nf == 2) { + conf.primary = f[1]; + } else if ((strcmp(f[0], "X") == 0 || strcmp(f[0], "S") == 0) && nf == 2) { + bool is_ext = f[0][0] == 'X'; + const char ***arr = is_ext ? &ext : &isx; + int *cnt = is_ext ? &next : &nisx; + const char **grown = + (const char **)cbm_arena_alloc(&x->arena, (size_t)(*cnt + 1) * sizeof(char *)); + if (!grown) { + return false; + } + if (*cnt > 0) { + memcpy(grown, *arr, (size_t)*cnt * sizeof(char *)); + } + grown[(*cnt)++] = f[1]; + *arr = grown; + } + } + if (!is_conf) { + return true; + } + if (x->nconfs >= *conf_cap) { + int ncap = *conf_cap ? *conf_cap * PAIR_LEN : RR_FIELDS; + rr_conf_t *grown = (rr_conf_t *)cbm_arena_alloc(&x->arena, (size_t)ncap * sizeof(*grown)); + if (!grown) { + return false; + } + if (x->nconfs > 0) { + memcpy(grown, x->confs, (size_t)x->nconfs * sizeof(*grown)); + } + x->confs = grown; + *conf_cap = ncap; + } + const char *b = rr_base(path); + conf.dir = cbm_arena_strndup(&x->arena, path, b > path ? (size_t)(b - path - SKIP_ONE) : 0); + if (!conf.dir) { + return false; + } + conf.primary = conf.primary ? conf.primary : "py"; + conf.ext = ext; + conf.next_count = next; + conf.isx = isx; + conf.nisx = nisx; + x->confs[x->nconfs++] = conf; + return true; +} + +static void *rr_build(const cbm_doclink_build_in_t *in) { + rr_index_t *x = (rr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x)); + if (!x) { + return NULL; + } + cbm_arena_init(&x->arena); + x->project = in->ctx ? in->ctx->project_name : NULL; + x->project_len = x->project ? strlen(x->project) : 0; + x->run_count = in->run_file_count; + x->run_paths = in->run_file_count > 0 + ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, + (size_t)in->run_file_count * sizeof(char *)) + : NULL; + x->mods = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->roots = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->aliases = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->modnode = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, RR_TABLE_INIT); + x->md = cbm_doclink_md_resolver.build(in); + if ((in->run_file_count > 0 && !x->run_paths) || !x->mods || !x->roots || !x->aliases || + !x->modnode || !x->md || !x->project || !rr_build_modules(x, in->graph)) { + rr_destroy(x); + return NULL; + } + int conf_cap = 0; + for (int i = 0; i < in->file_count; i++) { + const cbm_doclink_file_t *f = &in->files[i]; + if (f->run_file >= 0 && f->run_file < x->run_count) { + x->run_paths[f->run_file] = f->rel_path; /* valid until destroy */ + } + if (f->scope && strncmp(f->scope, CBM_DOCLINK_PY_SCOPE_TAG "\n", + strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n")) == 0) { + const char *path = cbm_arena_strdup(&x->arena, f->rel_path); + if (!path || !rr_read_scope(x, path, f->scope, &conf_cap)) { + rr_destroy(x); + return NULL; + } + } + } + if (x->nconfs > SKIP_ONE) { + qsort(x->confs, (size_t)x->nconfs, sizeof(*x->confs), rr_conf_cmp); + } + return x; +} + +static void rr_destroy(void *index) { + rr_index_t *x = (rr_index_t *)index; + if (!x) { + return; + } + if (x->md) { + cbm_doclink_md_resolver.destroy(x->md); + } + cbm_ht_free(x->mods); + cbm_ht_free(x->roots); + cbm_ht_free(x->aliases); + cbm_ht_free(x->modnode); + cbm_arena_destroy(&x->arena); + cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths); + cbm_free(CBM_MEM_CLASS_OTHER, x); +} + +/* The documentation set a document belongs to: the nearest conf.py at or + * above its directory; NULL for none. */ +static const rr_conf_t *rr_conf_for(const rr_index_t *x, const char *doc) { + char dir[RR_PATH_CAP]; + const char *b = rr_base(doc); + size_t dl = b > doc ? (size_t)(b - doc - SKIP_ONE) : 0; + if (dl >= sizeof(dir)) { + return NULL; + } + memcpy(dir, doc, dl); + dir[dl] = '\0'; + for (;;) { + rr_conf_t key = {.dir = dir}; + const rr_conf_t *hit = (const rr_conf_t *)bsearch(&key, x->confs, (size_t)x->nconfs, + sizeof(*x->confs), rr_conf_cmp); + if (hit) { + return hit; + } + if (!dir[0]) { + return NULL; + } + char *up = strrchr(dir, '/'); + if (up) { + *up = '\0'; + } else { + dir[0] = '\0'; + } + } +} + +static bool rr_listed(const char *const *list, int n, const char *s) { + for (int i = 0; i < n; i++) { + if (strcmp(list[i], s) == 0) { + return true; + } + } + return false; +} + +/* ── Outcomes ────────────────────────────────────────────────────── */ + +typedef enum { RR_NONE = 0, RR_HIT, RR_AMBIGUOUS, RR_TEST_ONLY } rr_kind_t; + +typedef struct { + rr_kind_t kind; + const cbm_gbuf_node_t *node; + bool exact; +} rr_res_t; + +static void rr_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *n, bool exact, + uint32_t first, uint32_t last) { + out->kind = CBM_DOCLINK_EDGE; + out->target = n; + out->exact = exact; + out->target_first = first; + out->target_last = last; +} + +static void rr_unresolved(cbm_doclink_outcome_t *out, int reason) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = reason; +} + +/* The outcome of a lookup. A name the document writes in full binds even + * test code (`django.test.TestCase` is Django's product); the name searches + * leave test code out themselves (R4). */ +static void rr_finish(cbm_doclink_outcome_t *out, rr_res_t r) { + if (r.kind == RR_HIT && r.node) { + rr_edge(out, r.node, r.exact, 0, 0); + } else if (r.kind == RR_AMBIGUOUS) { + rr_unresolved(out, CBM_DOCLINK_REASON_AMBIGUOUS); + } else if (r.kind == RR_TEST_ONLY) { + rr_unresolved(out, CBM_DOCLINK_REASON_TEST_ONLY); + } +} + +/* ── Python names ────────────────────────────────────────────────── */ + +static bool rr_word(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || + c >= 0x80; +} + +/* PY_TARGET_RE ^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$ */ +static bool rr_py_target(const char *t) { + bool start = true; + if (!t[0]) { + return false; + } + for (const char *p = t; *p; p++) { + unsigned char c = (unsigned char)*p; + if (c == '.') { + if (start) { + return false; + } + start = true; + continue; + } + if (start && c >= '0' && c <= '9') { + return false; + } + if (!rr_word(c)) { + return false; + } + start = false; + } + return !start; +} + +static const rr_mod_t *rr_module(const rr_index_t *x, const char *name, int *count) { + const rr_mod_t *m = (const rr_mod_t *)cbm_ht_get(x->mods, name); + *count = 0; + for (const rr_mod_t *k = m; k; k = k->next) { + (*count)++; + } + return m; +} + +/* One lookup of H3 resolve_dotted: the longest module prefix, the rest by + * qualified name. RR_NONE with `next` set: a package re-export names the + * dotted name the lookup continues with. */ +static rr_res_t rr_dotted_step(const rr_index_t *x, const cbm_gbuf_t *graph, const char *dotted, + char *next, size_t next_cap) { + rr_res_t none = {RR_NONE, NULL, false}; + next[0] = '\0'; + size_t dlen = strlen(dotted); + if (dlen == 0 || dlen >= RR_PATH_CAP) { + return none; + } + char buf[RR_PATH_CAP]; + memcpy(buf, dotted, dlen + SKIP_ONE); + size_t ends[RR_PIECES]; + int np = 0; + for (size_t i = 0; i <= dlen && np < RR_PIECES; i++) { + if (buf[i] == '.' || buf[i] == '\0') { + ends[np++] = i; + } + } + const rr_mod_t *mod = NULL; + int k = np; + for (; k > 0; k--) { + char c = buf[ends[k - 1]]; + buf[ends[k - 1]] = '\0'; + int count = 0; + mod = rr_module(x, buf, &count); + buf[ends[k - 1]] = c; + if (count > SKIP_ONE) { + return (rr_res_t){RR_AMBIGUOUS, NULL, false}; + } + if (mod) { + break; + } + } + if (!mod) { + return none; + } + const char *rest = ends[k - 1] < dlen ? dotted + ends[k - 1] + SKIP_ONE : ""; + if (!rest[0]) { + const cbm_gbuf_node_t *n = (const cbm_gbuf_node_t *)cbm_ht_get(x->modnode, mod->file); + if (!n && mod->is_pkg) { + char dir[RR_PATH_CAP]; + const char *b = rr_base(mod->file); + size_t l = (size_t)(b - mod->file - SKIP_ONE); + memcpy(dir, mod->file, l); + dir[l] = '\0'; + n = cbm_doclink_md_folder(x->md, dir); + } + return n ? (rr_res_t){RR_HIT, n, true} : none; + } + char qn[RR_PATH_CAP * PAIR_LEN]; + snprintf(qn, sizeof(qn), "%s.%s", mod->qn_base, rest); + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(graph, qn); + if (n) { + return (rr_res_t){RR_HIT, n, true}; + } + if (!mod->is_pkg) { + return none; + } + size_t first_len = strcspn(rest, "."); + for (const rr_alias_t *a = (const rr_alias_t *)cbm_ht_get(x->aliases, mod->file); a; + a = a->next) { + if (strlen(a->alias) != first_len || strncmp(a->alias, rest, first_len) != 0) { + continue; + } + /* the source module, absolute: a relative import counts from the + * package as written (H3 init_imports) */ + char pkg[RR_PATH_CAP]; + size_t pl = ends[k - 1]; + memcpy(pkg, dotted, pl); + pkg[pl] = '\0'; + char src[RR_PATH_CAP]; + if (a->level > 0) { + for (int up = a->level - SKIP_ONE; up > 0; up--) { + char *dot = strrchr(pkg, '.'); + if (!dot) { + pkg[0] = '\0'; + break; + } + *dot = '\0'; + } + snprintf(src, sizeof(src), "%s%s%s", pkg, pkg[0] && a->src[0] ? "." : "", a->src); + } else { + snprintf(src, sizeof(src), "%s", a->src); + } + const char *tail = rest + first_len; + int w = snprintf(next, next_cap, "%s%s%s%s", src, src[0] ? "." : "", a->orig, tail); + if (w <= 0 || (size_t)w >= next_cap) { + next[0] = '\0'; + } + return none; /* the lookup continues at the re-export's source */ + } + return none; +} + +/* H3 resolve_dotted: a module prefix, the rest by qualified name, a package's + * re-exports followed (at most RR_ALIAS_DEPTH of them, as H3). */ +static rr_res_t rr_dotted(const rr_index_t *x, const cbm_gbuf_t *graph, const char *dotted) { + rr_res_t none = {RR_NONE, NULL, false}; + char cur[RR_PATH_CAP]; + char next[RR_PATH_CAP]; + size_t n = strlen(dotted); + if (n >= sizeof(cur)) { + return none; + } + memcpy(cur, dotted, n + SKIP_ONE); + for (int hop = 0; hop <= RR_ALIAS_DEPTH; hop++) { + rr_res_t r = rr_dotted_step(x, graph, cur, next, sizeof(next)); + if (r.kind != RR_NONE || !next[0]) { + return r; + } + memcpy(cur, next, strlen(next) + SKIP_ONE); + } + return none; +} + +static const char *const RR_PY_LABELS[] = {"Class", "Function", "Method", "Variable", "Module", + "Field", "Enum", "Interface", "Type", NULL}; + +/* PY_KIND_LABELS: the labels a role or directive kind may bind by name. */ +static bool rr_kind_label(const char *kind, const char *label) { + static const struct { + const char *kind; + const char *labels[3]; + } map[] = {{"class", {"Class"}}, + {"exc", {"Class"}}, + {"exception", {"Class"}}, + {"meth", {"Method"}}, + {"method", {"Method"}}, + {"classmethod", {"Method"}}, + {"staticmethod", {"Method"}}, + {"func", {"Function"}}, + {"function", {"Function"}}, + {"decorator", {"Function"}}, + {"deco", {"Function"}}, + {"decoratormethod", {"Method"}}, + {"attr", {"Variable", "Method"}}, + {"attribute", {"Variable", "Method"}}, + {"property", {"Method", "Variable"}}, + {"data", {"Variable"}}, + {"const", {"Variable"}}, + {"mod", {"Module"}}, + {"module", {"Module"}}}; + for (size_t i = 0; i < sizeof(map) / sizeof(map[0]); i++) { + if (strcmp(map[i].kind, kind) != 0) { + continue; + } + for (int j = 0; j < 3 && map[i].labels[j]; j++) { + if (strcmp(map[i].labels[j], label) == 0) { + return true; + } + } + return false; + } + return true; /* obj: any code label */ +} + +static bool rr_in_labels(const char *label, const char *const *labels) { + for (int i = 0; label && labels[i]; i++) { + if (strcmp(label, labels[i]) == 0) { + return true; + } + } + return false; +} + +/* The QN prefix of module M's definitions, written as M; NULL when M is no + * module of the repository. */ +static const char *rr_module_base(const rr_index_t *x, const char *m) { + int count = 0; + const rr_mod_t *mod = m && m[0] ? rr_module(x, m, &count) : NULL; + return count == SKIP_ONE ? mod->qn_base : NULL; +} + +static bool rr_under(const char *qn, const char *base) { + size_t bl = strlen(base); + return strncmp(qn, base, bl) == 0 && qn[bl] == '.'; +} + +/* Kinds whose one-piece name may be searched: a class, exception, function or + * module-level datum is named by itself. A bare member (`:meth:`save``, + * `:attr:`name``) names a member of SOME class: the field test found 37 of 40 + * such search results wrong without a class context, so it never is. */ +static bool rr_bare_searchable(const char *kind) { + static const char *const kinds[] = {"class", "exc", "func", "data", "const", "deco", NULL}; + for (int i = 0; kinds[i]; i++) { + if (strcmp(kinds[i], kind) == 0) { + return true; + } + } + return false; +} + +/* H3 unique_candidates, with the field test's fixes: a bare member name is + * never searched (rr_bare_searchable); an object directive's target lies + * inside the module it is declared in (its one-piece name only there); + * test code is no target of a search (R4). */ +static rr_res_t rr_unique(const rr_index_t *x, const cbm_gbuf_t *graph, const char *t, + const char *kind, const char *m, bool directive) { + rr_res_t none = {RR_NONE, NULL, false}; + const char *last = strrchr(t, '.'); + const char *mbase = rr_module_base(x, m); + if (!last && !(directive ? mbase != NULL : rr_bare_searchable(kind))) { + return none; + } + const char *name = last ? last + SKIP_ONE : t; + char suffix[RR_PATH_CAP]; + if (snprintf(suffix, sizeof(suffix), ".%s", t) >= (int)sizeof(suffix)) { + return none; + } + size_t sl = strlen(suffix); + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return none; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *in_mod = NULL; + int in_mod_hits = 0; + int test_hits = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *qn = n->qualified_name; + if (!rr_in_labels(n->label, RR_PY_LABELS) || !rr_kind_label(kind, n->label) || + !n->file_path || !rr_ends(n->file_path, ".py") || !qn || strlen(qn) < sl || + strcmp(qn + strlen(qn) - sl, suffix) != 0) { + continue; + } + if (cbm_doclink_md_test_path(n->file_path)) { + test_hits++; + continue; + } + hit = n; + hits++; + if (mbase && rr_under(qn, mbase)) { + in_mod = n; + in_mod_hits++; + } + } + if (mbase && (directive || hits > SKIP_ONE) && in_mod_hits > 0) { + hit = in_mod; + hits = in_mod_hits; + } else if (directive && mbase) { + hits = 0; /* declared in its module: never one found outside it */ + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, false}; + } + if (hits > SKIP_ONE) { + return (rr_res_t){RR_AMBIGUOUS, NULL, false}; + } + return test_hits > 0 ? (rr_res_t){RR_TEST_ONLY, NULL, false} : none; +} + +/* H3 PyResolver.resolve: the forms, then the name search. */ +static rr_res_t rr_py_resolve(const rr_index_t *x, const cbm_gbuf_t *graph, const char *t, + const char *kind, const char *m, const char *c, bool refspecific, + bool directive) { + char forms[4][RR_PATH_CAP]; + int nf = 0; + bool has_m = m && m[0]; + bool has_c = c && c[0]; + if (refspecific) { + if (has_m && has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s.%s", m, c, t); + } + if (has_m) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", m, t); + } + snprintf(forms[nf++], RR_PATH_CAP, "%s", t); + } else { + snprintf(forms[nf++], RR_PATH_CAP, "%s", t); + if (has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", c, t); + } + if (has_m) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s", m, t); + } + if (has_m && has_c) { + snprintf(forms[nf++], RR_PATH_CAP, "%s.%s.%s", m, c, t); + } + } + /* a module answers only a module's kind: `:meth:`dispatch`` is never the + * package tests/dispatch (whose import name is `dispatch`) */ + bool module_kind = + strcmp(kind, "mod") == 0 || strcmp(kind, "module") == 0 || strcmp(kind, "obj") == 0; + for (int i = 0; i < nf; i++) { + rr_res_t r = rr_dotted(x, graph, forms[i]); + if (r.kind == RR_HIT && + (module_kind || !r.node->label || + (strcmp(r.node->label, "Module") != 0 && strcmp(r.node->label, "Folder") != 0))) { + return r; + } + } + return rr_unique(x, graph, t, kind, m, directive); +} + +/* Not found: the repository's name (a row) or another project's (LOCAL). */ +static void rr_py_unfound(const rr_index_t *x, const char *t, cbm_doclink_outcome_t *out) { + char root[RR_PATH_CAP]; + size_t rl = strcspn(t, "."); + if (!strchr(t, '.') || rl >= sizeof(root)) { + out->kind = CBM_DOCLINK_LOCAL; /* a bare name no context resolved */ + return; + } + memcpy(root, t, rl); + root[rl] = '\0'; + if (cbm_ht_get(x->roots, root)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + out->kind = CBM_DOCLINK_LOCAL; +} + +/* ── Token records ───────────────────────────────────────────────── */ + +typedef struct { + char buf[RR_RAW_CAP]; + char *f[RR_FIELDS]; + int n; +} rr_rec_t; + +static bool rr_record(const char *raw, rr_rec_t *r) { + size_t n = strlen(raw); + if (n >= sizeof(r->buf)) { + return false; + } + memcpy(r->buf, raw, n + SKIP_ONE); + r->n = rr_split(r->buf, r->f, RR_FIELDS); + for (int i = r->n; i < RR_FIELDS; i++) { + r->f[i] = ""; + } + return true; +} + +/* H3 split_domain. */ +static void rr_split_domain(const char *name, const char *primary, const char *const *py_members, + const char *const *c_members, char *dom, size_t cap, + const char **rest) { + const char *colon = strchr(name, ':'); + if (colon) { + size_t l = (size_t)(colon - name); + snprintf(dom, cap, "%.*s", (int)(l < cap ? l : cap - SKIP_ONE), name); + *rest = colon + SKIP_ONE; + return; + } + const char *const *members = strcmp(primary, "c") == 0 ? c_members + : strcmp(primary, "py") == 0 ? py_members + : NULL; + bool in = false; + for (int i = 0; members && members[i]; i++) { + in = in || strcmp(members[i], name) == 0; + } + snprintf(dom, cap, "%s", in ? primary : "std"); + *rest = name; +} + +static const char *const RR_PY_ROLES[] = {"func", "class", "meth", "attr", "mod", "exc", + "data", "obj", "const", "deco", NULL}; +static const char *const RR_C_ROLES[] = {"func", "macro", "struct", "union", "enum", "enumerator", + "type", "member", "data", "var", NULL}; +static const char *const RR_PY_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", NULL}; +static const char *const RR_PY_DOMAIN_OBJ[] = { + "class", "exception", "method", "classmethod", "staticmethod", + "attribute", "property", "function", "decorator", "decoratormethod", + "data", "module", "currentmodule", NULL}; +static const char *const RR_C_OBJ[] = {"function", "macro", "type", "struct", "union", + "enum", "enumerator", "member", "var", NULL}; +static const char *const RR_C_DOMAIN_OBJ[] = { + "function", "macro", "type", "struct", "union", "enum", "enumerator", + "member", "var", "namespace", "namespace-push", "namespace-pop", NULL}; + +static bool rr_member_of(const char *s, const char *const *list) { + for (int i = 0; list[i]; i++) { + if (strcmp(list[i], s) == 0) { + return true; + } + } + return false; +} + +/* H3 parse_target: title , then !, ~, a leading . and a trailing (). */ +typedef struct { + char t[RR_PATH_CAP]; + bool bang; + bool refspecific; +} rr_target_t; + +static bool rr_parse_target(const char *content, rr_target_t *out) { + char c[RR_PATH_CAP]; + size_t w = 0; + for (const char *p = content; *p && w + SKIP_ONE < sizeof(c); p++) { + if (*p == '\\' && (p[1] == ' ' || p[1] == '\t')) { + p++; /* an escaped space joins its neighbours */ + continue; + } + c[w++] = *p; + } + c[w] = '\0'; + const char *t = c; + size_t tl = w; + if (w > 0 && c[w - SKIP_ONE] == '>') { + char *lt = strrchr(c, '<'); + if (lt && lt > c) { + /* a title before it: ^(.*?)\s*<([^<>]+)>$ */ + size_t inner = (size_t)(c + w - SKIP_ONE - (lt + SKIP_ONE)); + if (inner > 0 && !memchr(lt + SKIP_ONE, '>', inner)) { + t = lt + SKIP_ONE; + tl = inner; + while (tl > 0 && t[0] == ' ') { + t++; + tl--; + } + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + tl--; + } + } + } + } + out->bang = false; + out->refspecific = false; + for (int round = 0; round < PAIR_LEN; round++) { + if (tl > 0 && t[0] == '!') { + out->bang = true; + t++; + tl--; + } + if (tl > 0 && t[0] == '~') { + t++; + tl--; + } + } + if (tl > 0 && t[0] == '.') { + out->refspecific = true; + t++; + tl--; + } + if (tl >= PAIR_LEN && t[tl - PAIR_LEN] == '(' && t[tl - SKIP_ONE] == ')') { + tl -= PAIR_LEN; + } + while (tl > 0 && t[0] == ' ') { + t++; + tl--; + } + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + tl--; + } + if (tl >= sizeof(out->t)) { + return false; + } + memcpy(out->t, t, tl); + out->t[tl] = '\0'; + return true; +} + +/* ── C domain ────────────────────────────────────────────────────── */ + +static bool rr_c_file(const char *path) { + static const char *const exts[] = {".c", ".h", ".cc", ".cpp", ".cxx", + ".hh", ".hpp", ".hxx", NULL}; + for (int i = 0; path && exts[i]; i++) { + if (rr_ends(path, exts[i])) { + return true; + } + } + return false; +} + +/* H3 c_is_def: a definition, not a usage or a forward declaration. */ +static bool rr_c_def(const cbm_gbuf_node_t *n) { + static const char *const defining[] = {"Function", "Type", "Variable", "Field", "Method", NULL}; + return rr_in_labels(n->label, defining) || n->end_line > n->start_line; +} + +/* Macro nodes are not proxied on the incremental route (pipeline_delta.c), so + * a C role never binds one: a re-resolved document binds what a full build + * does. */ +static const char *const RR_C_LABELS[] = {"Function", "Class", "Enum", "Struct", "Type", NULL}; + +static rr_res_t rr_c_name(const cbm_gbuf_t *graph, const char *name, bool data, + const char *in_file) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + rr_res_t none = {RR_NONE, NULL, false}; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return none; + } + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + const cbm_gbuf_node_t *def = NULL; + int defs = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + const char *fp = n->file_path; + bool label = rr_in_labels(n->label, RR_C_LABELS) || + (data && n->label && strcmp(n->label, "Variable") == 0) || + (n->label && strcmp(n->label, "Method") == 0 && fp && + (rr_ends(fp, ".c") || rr_ends(fp, ".h"))); + if (!label || !rr_c_file(fp) || (in_file && strcmp(fp, in_file) != 0) || + (!in_file && cbm_doclink_md_test_path(fp))) { + continue; + } + if (!in_file && !rr_c_def(n)) { + continue; + } + hit = n; + hits++; + if (rr_c_def(n)) { + def = n; + defs++; + } + } + if (in_file && hits > SKIP_ONE && defs == SKIP_ONE) { + return (rr_res_t){RR_HIT, def, true}; /* H3 kd_node: the one definition */ + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, in_file != NULL}; + } + return hits > SKIP_ONE ? (rr_res_t){RR_AMBIGUOUS, NULL, false} : none; +} + +/* A member through its struct: the one Field `struct.member`. */ +static rr_res_t rr_c_member(const cbm_gbuf_t *graph, const char *owner, const char *member) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + rr_res_t none = {RR_NONE, NULL, false}; + if (cbm_gbuf_find_by_name(graph, member, &nodes, &count) != 0) { + return none; + } + char suffix[RR_PATH_CAP]; + snprintf(suffix, sizeof(suffix), ".%s.%s", owner ? owner : "", member); + const cbm_gbuf_node_t *hit = NULL; + int hits = 0; + for (int i = 0; i < count; i++) { + const cbm_gbuf_node_t *n = nodes[i]; + if (!n->label || strcmp(n->label, "Field") != 0 || !rr_c_file(n->file_path) || + cbm_doclink_md_test_path(n->file_path)) { + continue; + } + if (owner && (!n->qualified_name || !rr_ends(n->qualified_name, suffix))) { + continue; + } + hit = n; + hits++; + } + if (hits == SKIP_ONE) { + return (rr_res_t){RR_HIT, hit, false}; + } + return hits > SKIP_ONE ? (rr_res_t){RR_AMBIGUOUS, NULL, false} : none; +} + +static bool rr_c_ident(const char *s) { + if (!s[0] || (s[0] >= '0' && s[0] <= '9')) { + return false; + } + for (const char *p = s; *p; p++) { + unsigned char c = (unsigned char)*p; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_')) { + return false; + } + } + return true; +} + +/* Is the name the repository's C code at all (a row rather than LOCAL)? */ +static bool rr_c_known(const cbm_gbuf_t *graph, const char *name) { + const cbm_gbuf_node_t **nodes = NULL; + int count = 0; + if (cbm_gbuf_find_by_name(graph, name, &nodes, &count) != 0) { + return false; + } + for (int i = 0; i < count; i++) { + if (rr_c_file(nodes[i]->file_path)) { + return true; + } + } + return false; +} + +/* H3 c_clean then c_role: struct/union/enum/typedef dropped, -> read as ., + * a trailing * or () dropped; a member through its struct. */ +static void rr_c_role(const cbm_gbuf_t *graph, const char *doc, const char *rname, + const char *target, cbm_doclink_outcome_t *out) { + char t[RR_PATH_CAP]; + const char *s = target; + static const char *const kws[] = {"struct ", "union ", "enum ", "typedef ", NULL}; + for (int i = 0; kws[i]; i++) { + if (strncmp(s, kws[i], strlen(kws[i])) == 0) { + s += strlen(kws[i]); + break; + } + } + size_t w = 0; + for (const char *p = s; *p && w + SKIP_ONE < sizeof(t); p++) { + if (p[0] == '-' && p[1] == '>') { + t[w++] = '.'; + p++; + } else { + t[w++] = *p; + } + } + while (w > 0 && (t[w - SKIP_ONE] == ' ' || t[w - SKIP_ONE] == '*')) { + w--; + } + t[w] = '\0'; + if (w >= PAIR_LEN && t[w - PAIR_LEN] == '(' && t[w - SKIP_ONE] == ')') { + t[w - PAIR_LEN] = '\0'; + } + char *start = t; + while (*start == ' ') { + start++; + } + rr_res_t r; + const char *name = start; + if (strcmp(rname, "member") == 0 || strchr(start, '.')) { + char *dot = strrchr(start, '.'); + const char *owner = NULL; + if (dot) { + *dot = '\0'; + char *prev = strrchr(start, '.'); + owner = prev ? prev + SKIP_ONE : start; + name = dot + SKIP_ONE; + } + if (!rr_c_ident(name) || (owner && !rr_c_ident(owner))) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + r = rr_c_member(graph, owner, name); + } else { + if (!rr_c_ident(name)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + bool data = strcmp(rname, "data") == 0 || strcmp(rname, "var") == 0; + r = rr_c_name(graph, name, data, NULL); + } + if (r.kind == RR_NONE) { + if (rr_c_known(graph, name)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + out->kind = CBM_DOCLINK_LOCAL; + } + return; + } + rr_finish(out, r); +} + +/* H3 c_sig_name: the declared name of a C object directive's signature. */ +static bool rr_c_sig_name(const char *dname, const char *sig, char *out, size_t cap) { + if (strcmp(dname, "function") == 0) { + size_t n = strlen(sig); + /* (*name)( : a function pointer */ + for (size_t i = 0; i < n; i++) { + if (sig[i] != '(') { + continue; + } + size_t j = i + SKIP_ONE; + while (j < n && sig[j] == ' ') { + j++; + } + if (j >= n || sig[j] != '*') { + continue; + } + j++; + while (j < n && sig[j] == ' ') { + j++; + } + size_t a = j; + while (j < n && (rr_word((unsigned char)sig[j]) && (unsigned char)sig[j] < 0x80)) { + j++; + } + size_t b = j; + while (j < n && sig[j] == ' ') { + j++; + } + if (b > a && j < n && sig[j] == ')') { + j++; + while (j < n && sig[j] == ' ') { + j++; + } + if (j < n && sig[j] == '(') { + snprintf(out, cap, "%.*s", (int)(b - a), sig + a); + return true; + } + } + } + /* ([A-Za-z_]\w*)\s*\( : the first identifier before a parenthesis */ + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)sig[i]; + bool start = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + if (!start || (i > 0 && rr_word((unsigned char)sig[i - SKIP_ONE]) && + (unsigned char)sig[i - SKIP_ONE] < 0x80)) { + continue; + } + size_t j = i; + while (j < n && rr_word((unsigned char)sig[j]) && (unsigned char)sig[j] < 0x80) { + j++; + } + size_t k = j; + while (k < n && sig[k] == ' ') { + k++; + } + if (k < n && sig[k] == '(') { + snprintf(out, cap, "%.*s", (int)(j - i), sig + i); + return true; + } + i = j; + } + return false; + } + const char *s = sig; + if (strcmp(dname, "macro") != 0) { + static const char *const kws[] = {"struct ", "union ", "enum ", "typedef ", NULL}; + for (int i = 0; kws[i]; i++) { + if (strncmp(s, kws[i], strlen(kws[i])) == 0) { + s += strlen(kws[i]); + break; + } + } + } + /* macro: the leading identifier; others: the last identifier */ + const char *best = NULL; + size_t best_len = 0; + for (const char *p = s; *p;) { + unsigned char c = (unsigned char)*p; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { + const char *e = p; + while (*e && rr_word((unsigned char)*e) && (unsigned char)*e < 0x80) { + e++; + } + best = p; + best_len = (size_t)(e - p); + if (strcmp(dname, "macro") == 0) { + break; + } + p = e; + } else { + if (strcmp(dname, "macro") == 0) { + break; + } + if (rr_word(c) && c < 0x80) { /* digits: part of no identifier start */ + while (*p && rr_word((unsigned char)*p) && (unsigned char)*p < 0x80) { + p++; + } + continue; + } + p++; + } + } + if (!best || best_len >= cap) { + return false; + } + memcpy(out, best, best_len); + out[best_len] = '\0'; + return true; +} + +/* ── Paths ───────────────────────────────────────────────────────── */ + +/* os.path.normpath(base/p) relative to the repository; false when it leaves + * the root. */ +static bool rr_norm(const char *base, const char *p, char *out, size_t cap) { + char tmp[RR_PATH_CAP * PAIR_LEN]; + snprintf(tmp, sizeof(tmp), "%s%s%s", base, base[0] && p[0] ? "/" : "", p); + const char *seg[RR_PIECES * PAIR_LEN]; + size_t len[RR_PIECES * PAIR_LEN]; + int n = 0; + for (char *s = tmp; *s;) { + char *e = strchr(s, '/'); + size_t l = e ? (size_t)(e - s) : strlen(s); + if (l == 0 || (l == SKIP_ONE && s[0] == '.')) { + /* nothing */ + } else if (l == PAIR_LEN && s[0] == '.' && s[1] == '.') { + if (n == 0) { + return false; + } + n--; + } else if (n < (int)(sizeof(seg) / sizeof(seg[0]))) { + seg[n] = s; + len[n] = l; + n++; + } else { + return false; + } + if (!e) { + break; + } + s = e + SKIP_ONE; + } + size_t w = 0; + for (int i = 0; i < n; i++) { + if (w + len[i] + PAIR_LEN > cap) { + return false; + } + if (i > 0) { + out[w++] = '/'; + } + memcpy(out + w, seg[i], len[i]); + w += len[i]; + } + out[w] = '\0'; + return true; +} + +static void rr_doc_dir(const char *doc, char *out, size_t cap) { + const char *b = rr_base(doc); + size_t l = b > doc ? (size_t)(b - doc - SKIP_ONE) : 0; + snprintf(out, cap, "%.*s", (int)(l < cap ? l : cap - SKIP_ONE), doc); +} + +/* The File node at `rel`; else a row when its directory is the repository's + * (doc rot), LOCAL otherwise. */ +static const cbm_gbuf_node_t *rr_file(const rr_index_t *x, const cbm_gbuf_t *graph, const char *rel, + bool folder_too, cbm_doclink_outcome_t *out) { + const cbm_gbuf_node_t *f = rel[0] ? cbm_pipeline_file_node(graph, x->project, rel) : NULL; + if (!f && folder_too && rel[0]) { + f = cbm_doclink_md_folder(x->md, rel); + } + if (f) { + return f; + } + char dir[RR_PATH_CAP]; + rr_doc_dir(rel, dir, sizeof(dir)); + if (dir[0] && cbm_doclink_md_folder(x->md, dir)) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + out->kind = CBM_DOCLINK_LOCAL; + } + return NULL; +} + +/* include / literalinclude: Sphinx reads `/x` from the documentation set's + * directory and everything else from the document's. */ +static bool rr_doc_path(const rr_index_t *x, const char *doc, const char *arg, char *out, + size_t cap) { + if (arg[0] == '/') { + const rr_conf_t *conf = rr_conf_for(x, doc); + return rr_norm(conf ? conf->dir : "", arg + SKIP_ONE, out, cap); + } + char dir[RR_PATH_CAP]; + rr_doc_dir(doc, dir, sizeof(dir)); + return rr_norm(dir, arg, out, cap); +} + +static void rr_literalinclude(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + const char *arg = r->f[1]; + if (!arg[0] || strchr(arg, '$')) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char rel[RR_PATH_CAP]; + if (!rr_doc_path(x, doc, arg, rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *file = rr_file(x, graph, rel, false, out); + if (!file) { + return; + } + const char *lines = r->f[2]; + const char *pyobject = r->f[3]; + if (pyobject[0]) { + /* H3: the definition `pyobject` of this .py file, by its QN */ + const cbm_gbuf_node_t *n = NULL; + if (rr_ends(rel, ".py")) { + char qn[RR_PATH_CAP * PAIR_LEN]; + char dotted[RR_PATH_CAP]; + size_t l = strlen(rel) - strlen(".py"); + snprintf(dotted, sizeof(dotted), "%.*s", (int)l, rel); + if (rr_ends(dotted, "/__init__")) { + dotted[strlen(dotted) - strlen("/__init__")] = '\0'; + } + for (char *c = dotted; *c; c++) { + *c = *c == '/' ? '.' : *c; + } + snprintf(qn, sizeof(qn), "%s.%s.%s", x->project, dotted, pyobject); + n = cbm_gbuf_find_by_qn(graph, qn); + } + if (n) { + rr_edge(out, n, true, 0, 0); + } else { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } + return; + } + uint32_t a = 0; + uint32_t b = 0; + for (const char *p = lines; *p;) { + if (*p >= '0' && *p <= '9') { + uint32_t v = 0; + while (*p >= '0' && *p <= '9') { + v = v < UINT32_MAX / 10 ? v * 10 + (uint32_t)(*p - '0') : UINT32_MAX; + p++; + } + a = (a == 0 || v < a) ? v : a; + b = v > b ? v : b; + } else { + p++; + } + } + if (a > 0) { + const cbm_gbuf_node_t *seg = cbm_doclink_md_segment(x->md, rel, a, b, NULL); + rr_edge(out, seg ? seg : file, true, a, b); + return; + } + rr_edge(out, file, true, 0, 0); +} + +/* kernel-doc: the file, or one of its names (H3 kd_node). */ +static void rr_kernel_doc(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + char rel[RR_PATH_CAP]; + if (!r->f[1][0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (!rr_norm("", r->f[1], rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *file = rr_file(x, graph, rel, false, out); + if (!file) { + return; + } + const char *name = r->f[2]; + if (!name[0]) { + rr_edge(out, file, true, 0, 0); + return; + } + rr_res_t res = rr_c_name(graph, name, true, rel); + if (res.kind == RR_NONE && rr_ends(rel, ".h")) { + /* the comment documents a prototype: its one definition elsewhere */ + res = rr_c_name(graph, name, false, NULL); + } + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +/* ── Resolve ─────────────────────────────────────────────────────── */ + +static void rr_py_role(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_conf_t *conf, const char *rname, const rr_rec_t *r, + cbm_doclink_outcome_t *out) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (tg.bang) { + out->kind = CBM_DOCLINK_LOCAL; /* `!x`: no link wanted */ + return; + } + /* an intersphinx name: `python:dict` */ + const char *colon = strchr(tg.t, ':'); + if (colon && conf) { + char key[RR_PATH_CAP]; + snprintf(key, sizeof(key), "%.*s", (int)(colon - tg.t), tg.t); + if (rr_listed(conf->isx, conf->nisx, key)) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + } + if (!rr_py_target(tg.t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + rr_res_t res = rr_py_resolve(x, graph, tg.t, rname, r->f[3], r->f[4], tg.refspecific, false); + if (res.kind == RR_NONE) { + rr_py_unfound(x, tg.t, out); + return; + } + rr_finish(out, res); +} + +static void rr_py_object(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const char *dname, const rr_rec_t *r, cbm_doclink_outcome_t *out) { + const char *m = r->f[3]; + if (strcmp(dname, "module") == 0) { + const char *t = r->f[2]; + if (!rr_py_target(t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + rr_res_t res = rr_dotted(x, graph, t); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); + return; + } + const char *fullname = r->f[6]; + if (!fullname[0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char full[RR_PATH_CAP]; + snprintf(full, sizeof(full), "%s%s%s", m, m[0] ? "." : "", fullname); + rr_res_t res = rr_dotted(x, graph, full); + if (res.kind != RR_HIT) { + res = rr_py_resolve(x, graph, fullname, dname, m, NULL, false, true); + } + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +static void rr_autodoc(const rr_index_t *x, const cbm_gbuf_t *graph, const char *doc, + const rr_rec_t *r, cbm_doclink_outcome_t *out) { + char t[RR_PATH_CAP]; + snprintf(t, sizeof(t), "%s", r->f[2]); + char *paren = strchr(t, '('); + if (paren) { + *paren = '\0'; + } + size_t tl = strlen(t); + while (tl > 0 && t[tl - SKIP_ONE] == ' ') { + t[--tl] = '\0'; + } + if (!rr_py_target(t)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + const char *m = r->f[3]; + char full[RR_PATH_CAP]; + if (strcmp(r->f[1], "automodule") == 0 || !m[0]) { + snprintf(full, sizeof(full), "%s", t); + } else { + snprintf(full, sizeof(full), "%s.%s", m, t); + } + rr_res_t res = rr_dotted(x, graph, full); + if (res.kind != RR_HIT && strcmp(full, t) != 0) { + res = rr_dotted(x, graph, t); + } + if (res.kind != RR_HIT) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + return; + } + rr_finish(out, res); +} + +static void rr_extlink(const rr_index_t *x, const cbm_gbuf_t *graph, const rr_rec_t *r, + cbm_doclink_outcome_t *out) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg) || !tg.t[0]) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + char rel[RR_PATH_CAP]; + const char *p = tg.t; + while (*p == '/') { + p++; + } + if (!rr_norm("", p, rel, sizeof(rel))) { + out->kind = CBM_DOCLINK_LOCAL; + return; + } + const cbm_gbuf_node_t *f = rr_file(x, graph, rel, true, out); + if (f) { + rr_edge(out, f, true, 0, 0); + } +} + +static void rr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link, + const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) { + (void)state; + const rr_index_t *x = (const rr_index_t *)index; + out->kind = CBM_DOCLINK_LOCAL; + const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL; + if (!doc || !link->raw) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (link->syntax == CBM_DOCLINK_MD_SUPERSEDES) { + /* a reST ADR's own supersedes statement (doc_adr.c) */ + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, link, graph, out); + return; + } + if (link->syntax == CBM_DOCLINK_RST_CODE_PATH || link->syntax == CBM_DOCLINK_RST_CODE_NAME) { + CBMDocLink md = *link; + md.syntax = link->syntax == CBM_DOCLINK_RST_CODE_PATH ? CBM_DOCLINK_MD_CODE_PATH + : CBM_DOCLINK_MD_CODE_NAME; + cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, &md, graph, out); + return; + } + rr_rec_t *r = (rr_rec_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*r)); + if (!r) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + return; + } + if (!rr_record(link->raw, r)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + cbm_free(CBM_MEM_CLASS_OTHER, r); + return; + } + const rr_conf_t *conf = rr_conf_for(x, doc); + const char *primary = conf ? conf->primary : "py"; + char dom[RR_PATH_CAP]; + const char *rest = ""; + switch (link->syntax) { + case CBM_DOCLINK_RST_ROLE: + rr_split_domain(r->f[1], primary, RR_PY_ROLES, RR_C_ROLES, dom, sizeof(dom), &rest); + if (strcmp(dom, "py") == 0 && rr_member_of(rest, RR_PY_ROLES)) { + rr_py_role(x, graph, doc, conf, rest, r, out); + } else if (strcmp(dom, "c") == 0 && rr_member_of(rest, RR_C_ROLES)) { + rr_target_t tg; + if (!rr_parse_target(r->f[2], &tg)) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (!tg.bang) { + rr_c_role(graph, doc, rest, tg.t, out); + } + } else if (strcmp(dom, "std") == 0 && conf && + rr_listed(conf->ext, conf->next_count, rest)) { + rr_extlink(x, graph, r, out); + } + break; + case CBM_DOCLINK_RST_OBJECT: + rr_split_domain(r->f[1], primary, RR_PY_DOMAIN_OBJ, RR_C_DOMAIN_OBJ, dom, sizeof(dom), + &rest); + if (strcmp(dom, "py") == 0 && rr_member_of(rest, RR_PY_OBJ)) { + rr_py_object(x, graph, doc, rest, r, out); + } else if (strcmp(dom, "c") == 0 && rr_member_of(rest, RR_C_OBJ)) { + char name[RR_PATH_CAP]; + if (!rr_c_sig_name(rest, r->f[2], name, sizeof(name))) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (strcmp(rest, "member") == 0) { + rr_res_t res = rr_c_member(graph, NULL, name); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + rr_finish(out, res); + } + } else { + bool data = strcmp(rest, "var") == 0; + rr_res_t res = rr_c_name(graph, name, data, NULL); + if (res.kind == RR_NONE) { + rr_unresolved(out, CBM_DOCLINK_REASON_MISSING); + } else { + rr_finish(out, res); + } + } + } + break; + case CBM_DOCLINK_RST_AUTODOC: + rr_autodoc(x, graph, doc, r, out); + break; + case CBM_DOCLINK_RST_INCLUDE: { + char rel[RR_PATH_CAP]; + const char *arg = r->f[2]; + bool ok = strcmp(r->f[1], "kernel-include") == 0 + ? rr_norm("", arg, rel, sizeof(rel)) + : rr_doc_path(x, doc, arg, rel, sizeof(rel)); + if (strchr(arg, '$')) { + rr_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); + } else if (ok) { + const cbm_gbuf_node_t *f = rr_file(x, graph, rel, false, out); + if (f) { + rr_edge(out, f, true, 0, 0); + } + } + break; + } + case CBM_DOCLINK_RST_LITERALINCLUDE: + rr_literalinclude(x, graph, doc, r, out); + break; + case CBM_DOCLINK_RST_KERNEL_DOC: + rr_kernel_doc(x, graph, doc, r, out); + break; + default: + break; + } + cbm_free(CBM_MEM_CLASS_OTHER, r); +} + +const cbm_doclink_resolver_t cbm_doclink_rst_resolver = { + .langs = {CBM_LANG_RST, CBM_LANG_PYTHON}, + .lang_count = 2, + .scope_tag = CBM_DOCLINK_PY_SCOPE_TAG, + .build = rr_build, + .destroy = rr_destroy, + .resolve = rr_resolve, + .via = "rst", +}; diff --git a/tests/test_doc_links_rst.c b/tests/test_doc_links_rst.c new file mode 100644 index 000000000..160dd5581 --- /dev/null +++ b/tests/test_doc_links_rst.c @@ -0,0 +1,440 @@ +/* + * test_doc_links_rst.c — reStructuredText documents -> Sections and MENTIONS + * edges (doclink_rst.c, doclink_py.c, doc_links_rst.c). + * + * The line model (titles, literal blocks, directives, comments), the tokens + * and their resolution contexts, the Python scope blob (package re-exports, + * Sphinx conf.py), the resolver's rules as the field test measured them + * (forms and re-exports, a module never answers a member's role, bare members + * are never searched, a name written in full binds even test code while a + * search leaves it out, intersphinx and other projects are no rows), the C + * domain, include / literalinclude / kernel-doc, extlinks, inline literals, + * Sphinx's source_suffix in discovery, reST ADRs, incremental == full. + */ +#include "../src/foundation/compat.h" +#include "test_framework.h" +#include "test_helpers.h" +#include "test_doc_mentions_helpers.h" + +#include "cbm.h" +#include "doclink.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" + +#include +#include +#include + +/* ── the scanner ─────────────────────────────────────────────────── */ + +static const char *const RST_DOC = + "Intro names :class:`pkg.Top` before any title.\n" /* 1 */ + "\n" /* 2 */ + "=========\n" /* 3 */ + " The API\n" /* 4 */ + "=========\n" /* 5 */ + "\n" /* 6 */ + ".. module:: pkg\n" /* 7 */ + "\n" /* 8 */ + "Use :class:`Model` and :meth:`~pkg.Model.save`::\n" /* 9 */ + "\n" /* 10 */ + " :class:`NotARole` in a literal block\n" /* 11 */ + " Fake Title\n" /* 12 */ + " ==========\n" /* 13 */ + "\n" /* 14 */ + ".. code-block:: python\n" /* 15 */ + "\n" /* 16 */ + " x = :func:`nope`\n" /* 17 */ + "\n" /* 18 */ + ".. class:: Model(name)\n" /* 19 */ + "\n" /* 20 */ + " .. method:: save(force=False)\n" /* 21 */ + "\n" /* 22 */ + " Saves; see :attr:`name`.\n" /* 23 */ + "\n" /* 24 */ + "Usage\n" /* 25 */ + "-----\n" /* 26 */ + "\n" /* 27 */ + ".. autofunction:: pkg.base.top\n" /* 28 */ + "\n" /* 29 */ + ".. literalinclude:: ../pkg/base.py\n" /* 30 */ + " :lines: 2-3\n" /* 31 */ + "\n" /* 32 */ + ".. include:: ../pkg/snippet.py\n" /* 33 */ + "\n" /* 34 */ + ".. include:: other.rst\n" /* 35 */ + "\n" /* 36 */ + ".. kernel-doc:: drivers/x.c\n" /* 37 */ + " :identifiers: foo bar\n" /* 38 */ + "\n" /* 39 */ + ".. a comment with :class:`Hidden`\n" /* 40 */ + "\n" /* 41 */ + "Text ``see :class:`Hidden2` there`` and ``pkg/base.py``,\n" /* 42 */ + "``pkg.base.top`` and :source:`a file `.\n"; /* 43 */ + +static int rst_count(const CBMFileResult *r, int syntax) { + int n = 0; + for (int i = 0; i < r->doc_links.count; i++) { + n += r->doc_links.items[i].syntax == syntax; + } + return n; +} + +static bool rst_raw_has(const CBMFileResult *r, const char *needle) { + for (int i = 0; i < r->doc_links.count; i++) { + if (strstr(r->doc_links.items[i].raw, needle)) { + return true; + } + } + return false; +} + +static const CBMDefinition *rst_def(const CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && strcmp(d->label, label) == 0 && d->name && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +TEST(rst_scan_structure) { + CBMFileResult *r = dm_extract(RST_DOC, CBM_LANG_RST, "docs/api.rst"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->doc_links.failed); + /* titles: over+under and under only; a title shape in a literal block is none */ + const CBMDefinition *api = rst_def(r, "Section", "The API"); + const CBMDefinition *usage = rst_def(r, "Section", "Usage"); + ASSERT_NOT_NULL(api); + ASSERT_NOT_NULL(usage); + ASSERT_NULL(rst_def(r, "Section", "Fake Title")); + ASSERT_EQ(api->start_line, 4); + ASSERT_EQ(api->end_line, 23); + ASSERT_EQ(usage->start_line, 25); + ASSERT_STR_EQ(api->qualified_name, "p.docs.api.The-API"); + ASSERT_NOT_NULL(api->docstring); + ASSERT_NOT_NULL(strstr(api->docstring, "Use :class:`Model`")); + /* nothing from a literal block, a code directive, a comment or an inline literal */ + ASSERT_FALSE(rst_raw_has(r, "NotARole")); + ASSERT_FALSE(rst_raw_has(r, "nope")); + ASSERT_FALSE(rst_raw_has(r, "Hidden")); + /* text before the first title belongs to the file */ + const CBMDocLink *top = dm_find_token(r, ":class:`pkg.Top`\tclass\tpkg.Top\t\t\t"); + ASSERT_NOT_NULL(top); + ASSERT_EQ(top->flags, CBM_DOCLINK_FLAG_FILE); + ASSERT_EQ(top->line, 1); + /* roles carry the module and class in force */ + const CBMDocLink *model = dm_find_token(r, ":class:`Model`\tclass\tModel\tpkg\t\t"); + ASSERT_NOT_NULL(model); + ASSERT_STR_EQ(model->source_qn, "p.docs.api.The-API"); + ASSERT_EQ(model->line, 9); + ASSERT_NOT_NULL(dm_find_token(r, ":attr:`name`\tattr\tname\tpkg\tModel\t")); + /* object directives: the signature and Python's full name in its class */ + ASSERT_NOT_NULL(dm_find_token(r, ".. class:: Model(name)\tclass\tModel(name)\tpkg\t\t\tModel")); + ASSERT_NOT_NULL(dm_find_token( + r, ".. method:: save(force=False)\tmethod\tsave(force=False)\tpkg\tModel\t\tModel.save")); + ASSERT_NOT_NULL(dm_find_token(r, ".. module:: pkg\tmodule\tpkg\tpkg\t\t\t")); + /* autodoc, literalinclude, an include of code (not of a document), kernel-doc names */ + ASSERT_NOT_NULL( + dm_find_token(r, ".. autofunction:: pkg.base.top\tautofunction\tpkg.base.top\tpkg\t")); + ASSERT_NOT_NULL( + dm_find_token(r, ".. literalinclude:: ../pkg/base.py\t../pkg/base.py\t2-3\t\t")); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_INCLUDE), 1); + ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_KERNEL_DOC), 3); + ASSERT_NOT_NULL(dm_find_token(r, ".. kernel-doc:: drivers/x.c\tdrivers/x.c\tbar")); + /* inline literals naming a path or a qualified name; a titled extlink */ + ASSERT_NOT_NULL(dm_find_token(r, "pkg/base.py")); + ASSERT_EQ(dm_find_token(r, "pkg/base.py")->syntax, CBM_DOCLINK_RST_CODE_PATH); + ASSERT_EQ(dm_find_token(r, "pkg.base.top")->syntax, CBM_DOCLINK_RST_CODE_NAME); + ASSERT_NOT_NULL( + dm_find_token(r, ":source:`a file `\tsource\ta file \tpkg\t\t")); + cbm_free_result(r); + PASS(); +} + +/* ── Python's scope ──────────────────────────────────────────────── */ + +TEST(rst_python_scope_blob) { + CBMFileResult *r = dm_extract("from .base import Model\n" + "from .sub.thing import helper as aid, other\n" + "from ..up import x\n" + "from os import *\n" + "from django.db import models\n" + "try:\n" + " from .opt import Nested\n" + "except ImportError:\n" + " pass\n", + CBM_LANG_PYTHON, "pkg/__init__.py"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(r->doc_scope); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "I\t1\tbase\tModel\tModel\n" + "I\t1\tsub.thing\thelper\taid\n" + "I\t1\tsub.thing\tother\tother\n" + "I\t2\tup\tx\tx\n" + "I\t0\tdjango.db\tmodels\tmodels\n"); + ASSERT_EQ(r->doc_links.count, 0); + cbm_free_result(r); + r = dm_extract("project = 'x'\n" + "primary_domain = 'c'\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + " \"sphinx\": (\"https://www.sphinx-doc.org/en/master/\", None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + " 'issue': ('https://github.com/o/r/issues/%s', '#%s'),\n" + "}\n" + "extlinks['tree'] = ('https://github.com/o/r/tree/main/%s', '%s')\n", + CBM_LANG_PYTHON, "docs/conf.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(r->doc_scope, "py1\n" + "C\n" + "D\tc\n" + "S\tpython\n" + "S\tsphinx\n" + "X\tsource\n" + "X\ttree\n"); + cbm_free_result(r); + r = dm_extract("from .x import y\n", CBM_LANG_PYTHON, "pkg/mod.py"); + ASSERT_NOT_NULL(r); + ASSERT_NULL(r->doc_scope); + cbm_free_result(r); + PASS(); +} + +/* ── the pipeline ────────────────────────────────────────────────── */ + +static const char *const API_RST = + "Models\n" /* 1 */ + "======\n" /* 2 */ + "\n" /* 3 */ + ".. module:: pkg\n" /* 4 */ + "\n" /* 5 */ + "Use :class:`Model`, :func:`pkg.aid` and :meth:`save`.\n" /* 6 */ + "Not :class:`python:dict`, :class:`otherlib.Thing`; gone\n" /* 7 */ + ":class:`pkg.Missing`.\n" /* 8 */ + "\n" /* 9 */ + ".. currentmodule:: pkg.views\n" /* 10 */ + "\n" /* 11 */ + ".. class:: View\n" /* 12 */ + "\n" /* 13 */ + " Calls :meth:`dispatch`.\n" /* 14 */ + "\n" /* 15 */ + "Testing\n" /* 16 */ + "=======\n" /* 17 */ + "\n" /* 18 */ + "Run :class:`pkg.test.runner.Runner`, not :class:`Runner`.\n" /* 19 */ + "\n" /* 20 */ + "Code\n" /* 21 */ + "====\n" /* 22 */ + "\n" /* 23 */ + "See :source:`pkg/base.py`, ``pkg.base.top`` and :c:func:`c_func`.\n" /* 24 */ + "\n" /* 25 */ + ".. autofunction:: pkg.base.top\n" /* 26 */ + "\n" /* 27 */ + ".. literalinclude:: ../pkg/base.py\n" /* 28 */ + " :lines: 2-3\n" /* 29 */ + "\n" /* 30 */ + ".. include:: /../pkg/sub/thing.py\n" /* 31 */ + "\n" /* 32 */ + ".. kernel-doc:: csrc/lib.c\n" /* 33 */ + " :identifiers: c_func\n"; /* 34 */ + +static void rst_write_repo(const char *repo, const char *init) { + th_write_file(TH_PATH(repo, "pkg/__init__.py"), init); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " def save(self):\n" + " return None\n" + "\n" + "\n" + "def top():\n" + " return 1\n"); + th_write_file(TH_PATH(repo, "pkg/other.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "pkg/sub/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/sub/thing.py"), "def helper():\n" + " return 2\n"); + th_write_file(TH_PATH(repo, "pkg/views.py"), "class View:\n" + " def dispatch(self):\n" + " return None\n"); + th_write_file(TH_PATH(repo, "pkg/test/__init__.py"), ""); + th_write_file(TH_PATH(repo, "pkg/test/runner.py"), "class Runner:\n" + " pass\n"); + /* tests/ is no package: tests/dispatch's import name is `dispatch` */ + th_write_file(TH_PATH(repo, "tests/dispatch/__init__.py"), ""); + th_write_file(TH_PATH(repo, "tests/dispatch/test_x.py"), "def test_x():\n" + " return None\n"); + th_write_file(TH_PATH(repo, "csrc/lib.c"), "int c_func(void) {\n" + " return 0;\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/conf.py"), + "project = 'x'\n" + "source_suffix = {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}\n" + "intersphinx_mapping = {\n" + " 'python': ('https://docs.python.org/3', None),\n" + "}\n" + "extlinks = {\n" + " 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n" + "}\n"); + th_write_file(TH_PATH(repo, "docs/api.rst"), API_RST); + th_write_file(TH_PATH(repo, "docs/topics/guide.txt"), "Guide\n" + "=====\n" + "\n" + "The :class:`pkg.views.View`.\n"); + th_write_file(TH_PATH(repo, "notes/readme.txt"), "Title\n" + "=====\n" + "\n" + "Plain :class:`pkg.views.View`.\n"); +} + +static const char *const INIT_PY = "from .base import Model\n" + "from .sub.thing import helper as aid\n"; + +static bool rst_edge(const char *db, const char *src, const char *tgt, const char *needle) { + char props[512]; + int n = 0; + dm_edge(db, src, tgt, props, sizeof(props), &n); + return n == 1 && (!needle || strstr(props, needle)); +} + +static int rst_rows(const char *db, const char *raw_prefix, const char *reason) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE '%s%%' AND reason='%s'", + raw_prefix, reason); + return dm_count(db, sql); +} + +TEST(rst_links_pipeline) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* a re-export through the module context: pkg.Model is pkg/base.py's */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", + "{\"via\":\"rst\",\"syntax\":\"role\",\"tier\":\"exact\",\"line\":6")); + /* an aliased re-export: pkg.aid is helper */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.sub.thing.helper", "\"tier\":\"exact\"")); + /* a bare member is never searched; intersphinx and another project: no rows */ + ASSERT_EQ(dm_count(db, + "SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE ':meth:`save`%' " + "OR raw LIKE ':class:`python:dict`%' OR raw LIKE " + "':class:`otherlib.Thing`%'"), + 0); + ASSERT_EQ(rst_rows(db, ":class:`pkg.Missing`", "missing"), 1); + /* a module never answers :meth: -- the class context does */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View.dispatch", "\"tier\":\"exact\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE " + "e.type='MENTIONS' AND t.file_path LIKE 'tests/%'"), + 0); + /* the object directive declares View in its module */ + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\"")); + /* a name written in full binds test code; a search for it does not */ + ASSERT_TRUE(rst_edge(db, "docs.api.Testing", "pkg.test.runner.Runner", "\"tier\":\"exact\"")); + ASSERT_EQ(rst_rows(db, ":class:`Runner`", "test_only_target"), 1); + /* extlink, inline literal, C domain, autodoc */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.py.__file__", "\"syntax\":\"role\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.top", "\"syntax\":\"code_name\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c_func", NULL)); + /* literalinclude :lines: the innermost definition, the lines on the edge */ + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]")); + /* include from the documentation set's directory; kernel-doc file */ + ASSERT_TRUE( + rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", "\"syntax\":\"include\"")); + ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c.__file__", "\"syntax\":\"kernel_doc\"")); + /* conf.py's source_suffix: .txt is reST in its documentation set only */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND " + "file_path='docs/topics/guide.txt' AND name='Guide'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.topics.guide.Guide", "pkg.views.View", "\"via\":\"rst\"")); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE file_path='notes/readme.txt'"), 0); + /* back navigation: the class knows the sections that name it */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN " + "nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND " + "t.name='View' AND t.label='Class' AND s.label='Section'"), + 2); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_adr_record) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_adr_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/g.db", tmp); + th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n" + " pass\n"); + th_write_file(TH_PATH(repo, "docs/decisions/0002-use-models.rst"), "2. Use models\n" + "=============\n" + "\n" + "Status\n" + "------\n" + "\n" + "Accepted\n" + "\n" + "Context\n" + "-------\n" + "\n" + "We store records.\n" + "\n" + "Decision\n" + "--------\n" + "\n" + "Use ``pkg.base.Model``.\n"); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='ADR' AND name='ADR-2' AND " + "json_extract(properties,'$.status')='accepted'"), + 1); + ASSERT_TRUE(rst_edge(db, "docs.decisions.0002-use-models.Decision", "pkg.base.Model", + "\"syntax\":\"code_name\"")); + th_cleanup(tmp); + PASS(); +} + +TEST(rst_links_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + char db[512]; + char full_db[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + rst_write_repo(repo, INIT_PY); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + /* the document changes alone: it re-resolves through the stored re-exports */ + th_write_file(TH_PATH(repo, "docs/api.rst"), "Models\n" + "======\n" + "\n" + ".. module:: pkg\n" + "\n" + "Only :class:`Model` and :func:`aid`.\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", NULL)); + /* the package re-exports another Model: the unchanged document follows */ + th_write_file(TH_PATH(repo, "pkg/__init__.py"), "from .other import Model\n" + "from .sub.thing import helper as aid\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "re-export change", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0); + ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.other.Model", NULL)); + th_cleanup(tmp); + PASS(); +} + +SUITE(doc_links_rst) { + RUN_TEST(rst_scan_structure); + RUN_TEST(rst_python_scope_blob); + RUN_TEST(rst_links_pipeline); + RUN_TEST(rst_adr_record); + RUN_TEST(rst_links_incremental); +} diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 7818c4c44..7246e5b95 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -195,7 +195,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"css", "Module:1"}, {"scss", "Module:1,Variable:1"}, {"markdown", "Module:1,Section:1"}, - {"rst", "Module:1"}, + {"rst", "Module:1,Section:1"}, {"dockerfile", "Module:1"}, {"makefile", "Function:1,Module:1"}, {"cmake", "Function:1,Module:1"}, diff --git a/tests/test_grammar_probe_g.c b/tests/test_grammar_probe_g.c index d0f99d4e0..215b4acee 100644 --- a/tests/test_grammar_probe_g.c +++ b/tests/test_grammar_probe_g.c @@ -399,12 +399,13 @@ TEST(probe_markdown_no_class_variable) { /* ══════════════════════════════════════════════════════════════════ * GROUP 6 — RST (.rst) * - * RST golden histogram: Module:1 (pure-data). - * reStructuredText headings are NOT extracted as Section nodes (unlike Markdown). + * RST golden histogram: Module:1,Section:1. + * reStructuredText titles are Section nodes, like Markdown headings + * (doclink_rst.c). * ══════════════════════════════════════════════════════════════════ */ -/* RST: document with sections → Module:1, no Section nodes. */ -TEST(probe_rst_module_only) { +/* RST: a document with two titles → Module:1 and a Section per title. */ +TEST(probe_rst_sections) { GpgMetrics m = gpg_metrics("docs.rst", "Project Title\n" "=============\n" "\n" @@ -415,9 +416,9 @@ TEST(probe_rst_module_only) { "\n" "Run ``pip install mypackage``.\n"); ASSERT_TRUE(m.ok); - /* GREEN: RST produces only a Module node (no Section nodes extracted). */ + /* GREEN: a Module node and the two titles' Section nodes. */ ASSERT_TRUE(m.modules == 1); - ASSERT_TRUE(m.sections == 0); + ASSERT_TRUE(m.sections == 2); PASS(); } @@ -1063,7 +1064,7 @@ SUITE(grammar_probe_g) { RUN_TEST(probe_markdown_no_class_variable); /* RST */ - RUN_TEST(probe_rst_module_only); + RUN_TEST(probe_rst_sections); RUN_TEST(probe_rst_no_crash); /* JSON */ diff --git a/tests/test_main.c b/tests/test_main.c index cad85c4ba..542df5b48 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1823,6 +1823,7 @@ extern void suite_doc_mentions_cost(void); extern void suite_doc_mentions_msbuild(void); extern void suite_doc_links_md(void); extern void suite_doc_links_pdf(void); +extern void suite_doc_links_rst(void); extern void suite_infrascan(void); extern void suite_cli(void); extern void suite_agent_clients(void); @@ -2292,6 +2293,7 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(doc_mentions_msbuild); RUN_SELECTED_SUITE(doc_links_md); RUN_SELECTED_SUITE(doc_links_pdf); + RUN_SELECTED_SUITE(doc_links_rst); /* Infrastructure scanning */ RUN_SELECTED_SUITE(infrascan);