mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
Martin Vogel
f403f921dd
A PDF page's Section carried the whole page text as its docstring, which is also what get_code_snippet returns for it: a crafted text layer stored megabytes per node and handed them to an agent whole (D6 of the document layer security review). Markdown, reStructuredText and AsciiDoc sections keep a short docstring already. A page's docstring is now its text up to 256 KiB, cut on a character boundary and followed by a line that says where it was cut and how long the text is. Pages hold a few kilobytes; the mentions are still read from the whole text. Test: pdf_page_doc_bounded: a 315 KB page keeps 256 KiB plus the note, and a path written at its end is still a mention (the whole text is stored with the bound reverted). Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1393 lines
43 KiB
C
1393 lines
43 KiB
C
/*
|
||
* doc_pdf.c — PDF documents in the graph.
|
||
*
|
||
* A PDF's text layer (pdf/) becomes one Section per page: name "page N", the
|
||
* page's text as its docstring, so search finds what a page says. The code a
|
||
* page names becomes doc-link tokens for the resolver (doc_links_pdf.c):
|
||
* STRUCTURAL mentions only -- paths ("src/auth/handler.go", "./x/y",
|
||
* "Pool.sol") and qualified names ("pkg.Type.method", "a::b", "x->y"). Bare
|
||
* identifiers never become tokens (field test E6c: bare names stay
|
||
* suggestions).
|
||
*
|
||
* The scanner is the field-tested one (E6 idents.py), ported with CPython's
|
||
* semantics: NFKC per code point plus canonical composition, the re module's
|
||
* leftmost-first matching, str predicates from the same Unicode tables
|
||
* (pdf/pdf_unicode.c). A mention split across two lines ("src/auth/" /
|
||
* "handler.go", "Handler::" / "run") is proposed as a JOIN token that lists
|
||
* the line-local tokens it would replace; the resolver keeps the join only
|
||
* when it resolves EXACT or UNIQUE, and the fragments stand otherwise.
|
||
*
|
||
* Hygiene from the field test, applied here where the text decides it: R1 a
|
||
* PDF under a test or fixture directory gives no tokens; R3 a mention written
|
||
* as a keyword argument ("(x, foo.bar=1)") is no reference.
|
||
*/
|
||
#include "doclink.h"
|
||
|
||
#include "arena.h"
|
||
#include "cbm.h"
|
||
#include "foundation/mem_core.h"
|
||
#include "helpers.h"
|
||
#include "pdf/pdf.h"
|
||
#include "pdf/pdf_unicode.h"
|
||
|
||
#include <stdatomic.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
|
||
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
|
||
/* candidates a join compared, for the cost test */
|
||
static atomic_size_t g_join_steps;
|
||
#endif
|
||
|
||
enum {
|
||
PDF_R3_WINDOW = 400, /* the field test looked this far back for "(" or "," */
|
||
PDF_PAGE_NAME = 32,
|
||
PDF_LIST_MIN = 64,
|
||
PDF_U8_MAX = 4,
|
||
PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */
|
||
};
|
||
|
||
#define PDF_JOIN_SEP '\x1f' /* between a join's forms */
|
||
#define PDF_JOIN_FRAG '\x1e' /* before a join's fragment token indices */
|
||
|
||
/* ── Word lists (the field test's, verbatim) ─────────────────────── */
|
||
|
||
static const char *const CODE_EXT[] = {
|
||
"py", "pyi", "pyx", "pxd", "c", "h", "cc", "cpp", "cxx", "hpp",
|
||
"hh", "hxx", "cs", "csx", "java", "kt", "kts", "scala", "go", "rs",
|
||
"rb", "php", "js", "jsx", "mjs", "cjs", "ts", "tsx", "swift", "m",
|
||
"mm", "sol", "sh", "bash", "zsh", "ps1", "psm1", "lua", "r", "jl",
|
||
"dart", "ex", "exs", "erl", "hs", "ml", "fs", "fsx", "vb", "sql",
|
||
"proto", "graphql", "gql", "thrift", "yaml", "yml", "json", "toml", "ini", "cfg",
|
||
"conf", "xml", "html", "htm", "css", "scss", "less", "md", "rst", "txt",
|
||
"adoc", "ipynb", "gradle", "csproj", "sln", "props", "targets", "cmake", "mk", "tf",
|
||
"hcl", "bzl", "lock", "env", "pdf", "vue", "svelte", "tex", "bib", "jsonl",
|
||
"csv", "tsv", "pom", "jar", "war", "dll", "so", "wasm", "zig", "nim",
|
||
"v", NULL};
|
||
|
||
static const char *const TLDS[] = {"com", "org", "net", "io", "dev", "edu", "gov", "de", "uk",
|
||
"ai", "co", "info", "app", "eu", "fr", "ch", "us", "ca",
|
||
"au", "jp", "cn", "ru", "nl", "se", "no", "fi", "it",
|
||
"es", "at", "be", "biz", "me", "tv", "ly", "gg", "sh",
|
||
"xyz", "cloud", "tech", "blog", NULL};
|
||
|
||
static const char *const SLASH_STOP[] = {
|
||
"and/or", "or/and", "input/output", "i/o", "read/write", "yes/no", "on/off",
|
||
"true/false", "he/she", "his/her", "s/he", "w/o", "n/a", "km/h",
|
||
"m/s", "tcp/ip", "client/server", "him/her", "either/or", NULL};
|
||
|
||
/* the field test's FIX: directories whose PDFs are fixtures (R1) */
|
||
static const char *const FIXTURE_DIRS[] = {"test", "tests", "cypress", "baseline_images",
|
||
"resources", "samples", "e2e", "fixtures",
|
||
"testdata", "test_data", "data", "examples",
|
||
NULL};
|
||
|
||
/* ── Code-point buffers ──────────────────────────────────────────── */
|
||
|
||
typedef struct {
|
||
uint32_t *v;
|
||
int n;
|
||
int cap;
|
||
bool fail;
|
||
} u32v_t;
|
||
|
||
static bool u32_push(u32v_t *b, uint32_t c) {
|
||
if (b->fail) {
|
||
return false;
|
||
}
|
||
if (b->n == b->cap) {
|
||
int ncap = b->cap ? b->cap * 2 : PDF_LIST_MIN;
|
||
uint32_t *g =
|
||
(uint32_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->v, (size_t)ncap * sizeof(*g));
|
||
if (!g) {
|
||
b->fail = true;
|
||
return false;
|
||
}
|
||
b->v = g;
|
||
b->cap = ncap;
|
||
}
|
||
b->v[b->n++] = c;
|
||
return true;
|
||
}
|
||
|
||
static void u32_free(u32v_t *b) {
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, b->v);
|
||
memset(b, 0, sizeof(*b));
|
||
}
|
||
|
||
/* ── normalise(): NFKC, zero-width characters dropped, hyphens unified ── */
|
||
|
||
static const pdf_nfkc_t *nfkc_find(uint32_t cp) {
|
||
int lo = 0;
|
||
int hi = PDF_NFKC_COUNT - 1;
|
||
while (lo <= hi) {
|
||
int mid = lo + (hi - lo) / 2;
|
||
if (PDF_NFKC[mid].cp < cp) {
|
||
lo = mid + 1;
|
||
} else if (PDF_NFKC[mid].cp > cp) {
|
||
hi = mid - 1;
|
||
} else {
|
||
return &PDF_NFKC[mid];
|
||
}
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
static bool ucomp(uint32_t a, uint32_t b, uint32_t *out) {
|
||
/* Hangul, algorithmically */
|
||
enum {
|
||
SBASE = 0xAC00,
|
||
LBASE = 0x1100,
|
||
VBASE = 0x1161,
|
||
TBASE = 0x11A7,
|
||
LCOUNT = 19,
|
||
VCOUNT = 21,
|
||
TCOUNT = 28,
|
||
NCOUNT = VCOUNT * TCOUNT,
|
||
SCOUNT = LCOUNT * NCOUNT
|
||
};
|
||
if (a >= LBASE && a < LBASE + LCOUNT && b >= VBASE && b < VBASE + VCOUNT) {
|
||
*out = SBASE + ((a - LBASE) * VCOUNT + (b - VBASE)) * TCOUNT;
|
||
return true;
|
||
}
|
||
if (a >= SBASE && a < SBASE + SCOUNT && (a - SBASE) % TCOUNT == 0 && b > TBASE &&
|
||
b < TBASE + TCOUNT) {
|
||
*out = a + (b - TBASE);
|
||
return true;
|
||
}
|
||
int lo = 0;
|
||
int hi = PDF_UCOMP_COUNT - 1;
|
||
while (lo <= hi) {
|
||
int mid = lo + (hi - lo) / 2;
|
||
const pdf_ucomp_t *e = &PDF_UCOMP[mid];
|
||
if (e->first < a || (e->first == a && e->second < b)) {
|
||
lo = mid + 1;
|
||
} else if (e->first > a || e->second > b) {
|
||
hi = mid - 1;
|
||
} else {
|
||
*out = e->composite;
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static bool normalise(const char *s, size_t n, u32v_t *out) {
|
||
size_t i = 0;
|
||
while (i < n) {
|
||
unsigned char c = (unsigned char)s[i];
|
||
uint32_t cp = c;
|
||
size_t k = 1;
|
||
if (c >= 0xF0 && i + 3 < n) {
|
||
cp = ((uint32_t)(c & 0x07) << 18) | ((uint32_t)(s[i + 1] & 0x3F) << 12) |
|
||
((uint32_t)(s[i + 2] & 0x3F) << 6) | (uint32_t)(s[i + 3] & 0x3F);
|
||
k = 4;
|
||
} else if (c >= 0xE0 && i + 2 < n) {
|
||
cp = ((uint32_t)(c & 0x0F) << 12) | ((uint32_t)(s[i + 1] & 0x3F) << 6) |
|
||
(uint32_t)(s[i + 2] & 0x3F);
|
||
k = 3;
|
||
} else if (c >= 0xC0 && i + 1 < n) {
|
||
cp = ((uint32_t)(c & 0x1F) << 6) | (uint32_t)(s[i + 1] & 0x3F);
|
||
k = 2;
|
||
}
|
||
i += k;
|
||
const pdf_nfkc_t *m = nfkc_find(cp);
|
||
int cnt = m ? m->len : 1;
|
||
for (int j = 0; j < cnt; j++) {
|
||
uint32_t x = m ? PDF_NFKC_SEQ[m->off + j] : cp;
|
||
uint32_t comp;
|
||
if (out->n > 0 && ucomp(out->v[out->n - 1], x, &comp)) {
|
||
out->v[out->n - 1] = comp;
|
||
} else if (!u32_push(out, x)) {
|
||
return false;
|
||
}
|
||
}
|
||
}
|
||
/* zero-width characters out; U+2010/U+2011 -> '-' */
|
||
int k = 0;
|
||
for (int j = 0; j < out->n; j++) {
|
||
uint32_t x = out->v[j];
|
||
if (x == 0x200B || x == 0x200C || x == 0x200D || x == 0x2060 || x == 0xFEFF) {
|
||
continue;
|
||
}
|
||
if (x == 0x2010 || x == 0x2011) {
|
||
x = '-';
|
||
}
|
||
out->v[k++] = x;
|
||
}
|
||
out->n = k;
|
||
return true;
|
||
}
|
||
|
||
/* ── Character classes of the field test's regexes ───────────────── */
|
||
|
||
static bool is_alnum_ascii(uint32_t c) {
|
||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
|
||
}
|
||
|
||
static bool is_alpha_ascii(uint32_t c) {
|
||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||
}
|
||
|
||
/* [A-Za-z0-9_.$@\-] */
|
||
static bool path_seg(uint32_t c) {
|
||
return is_alnum_ascii(c) || c == '_' || c == '.' || c == '$' || c == '@' || c == '-';
|
||
}
|
||
|
||
/* [A-Za-z0-9_.\-] */
|
||
static bool path_tail(uint32_t c) {
|
||
return is_alnum_ascii(c) || c == '_' || c == '.' || c == '-';
|
||
}
|
||
|
||
/* [A-Za-z0-9_] */
|
||
static bool path_end(uint32_t c) {
|
||
return is_alnum_ascii(c) || c == '_';
|
||
}
|
||
|
||
/* [A-Za-z_$] */
|
||
static bool ident_start(uint32_t c) {
|
||
return is_alpha_ascii(c) || c == '_' || c == '$';
|
||
}
|
||
|
||
/* [\w$] */
|
||
static bool ident_cont(uint32_t c) {
|
||
return pdf_u_word(c) || c == '$';
|
||
}
|
||
|
||
/* [A-Za-z0-9_$] (IDC) */
|
||
static bool idc(uint32_t c) {
|
||
return is_alnum_ascii(c) || c == '_' || c == '$';
|
||
}
|
||
|
||
/* ── Candidates ──────────────────────────────────────────────────── */
|
||
|
||
typedef enum {
|
||
MK_PATH = 0,
|
||
MK_PATH1,
|
||
MK_FILE,
|
||
MK_QUAL,
|
||
MK_SNAKE,
|
||
MK_CAMEL,
|
||
MK_CALL,
|
||
} mention_kind_t;
|
||
|
||
typedef struct {
|
||
int start;
|
||
int end;
|
||
mention_kind_t kind;
|
||
int norm_start; /* norm = a rewrite of the surface, kept in a pool */
|
||
int norm_len;
|
||
} cand_t;
|
||
|
||
typedef struct {
|
||
cand_t *v;
|
||
int n;
|
||
int cap;
|
||
bool fail;
|
||
u32v_t pool; /* norms */
|
||
} candv_t;
|
||
|
||
static bool cand_push(candv_t *cv, cand_t c) {
|
||
if (cv->fail) {
|
||
return false;
|
||
}
|
||
if (cv->n == cv->cap) {
|
||
int ncap = cv->cap ? cv->cap * 2 : PDF_LIST_MIN;
|
||
cand_t *g = (cand_t *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, cv->v, (size_t)ncap * sizeof(*g));
|
||
if (!g) {
|
||
cv->fail = true;
|
||
return false;
|
||
}
|
||
cv->v = g;
|
||
cv->cap = ncap;
|
||
}
|
||
cv->v[cv->n++] = c;
|
||
return true;
|
||
}
|
||
|
||
static void candv_free(candv_t *cv) {
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, cv->v);
|
||
u32_free(&cv->pool);
|
||
memset(cv, 0, sizeof(*cv));
|
||
}
|
||
|
||
static bool span_free(const uint8_t *taken, int a, int b) {
|
||
for (int i = a; i < b; i++) {
|
||
if (taken[i]) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
static void span_take(uint8_t *taken, int a, int b) {
|
||
memset(taken + a, 1, (size_t)(b - a));
|
||
}
|
||
|
||
/* URL_RE: (?:https?|ftp|file)://\S+ | www\.\S+ | \S+@\S+\.[A-Za-z]{2,} */
|
||
typedef struct {
|
||
int run_start; /* the non-space run the email alternative last looked at */
|
||
int run_end;
|
||
int at; /* its last '@' with a domain after it, or -1 */
|
||
int at_end; /* where that match ends */
|
||
} url_cache_t;
|
||
|
||
static bool lit_at(const uint32_t *t, int n, int p, const char *lit) {
|
||
for (int i = 0; lit[i]; i++) {
|
||
if (p + i >= n || t[p + i] != (uint32_t)(unsigned char)lit[i]) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
static int run_end_from(const uint32_t *t, int n, int p) {
|
||
while (p < n && !pdf_u_space(t[p])) {
|
||
p++;
|
||
}
|
||
return p;
|
||
}
|
||
|
||
static void url_cache_run(const uint32_t *t, int n, int p, url_cache_t *uc) {
|
||
if (p >= uc->run_start && p < uc->run_end) {
|
||
return; /* the run is known: validity of an '@' does not depend on p */
|
||
}
|
||
/* the scan enters a run at its start (every earlier position of a run
|
||
* filled this cache); scanning back is only for a caller that did not */
|
||
int rs = p;
|
||
while (rs > 0 && !pdf_u_space(t[rs - 1])) {
|
||
rs--;
|
||
}
|
||
uc->run_start = rs;
|
||
uc->run_end = run_end_from(t, n, p);
|
||
uc->at = -1;
|
||
uc->at_end = -1;
|
||
int re = uc->run_end;
|
||
/* the largest '.' followed by two ASCII letters, scanning from the right;
|
||
* for each '@' (right to left) the domain is the largest such '.' >= at+2 */
|
||
int best_dot = -1;
|
||
for (int a = re - 1; a > rs; a--) {
|
||
if (a + 2 < re && t[a] == '.' && is_alpha_ascii(t[a + 1]) && is_alpha_ascii(t[a + 2])) {
|
||
if (best_dot < 0) {
|
||
best_dot = a;
|
||
}
|
||
}
|
||
if (t[a] == '@' && best_dot >= a + 2) {
|
||
int e = best_dot + 1;
|
||
while (e < re && is_alpha_ascii(t[e])) {
|
||
e++;
|
||
}
|
||
uc->at = a;
|
||
uc->at_end = e;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
static int url_at(const uint32_t *t, int n, int p, url_cache_t *uc) {
|
||
if (lit_at(t, n, p, "https://") || lit_at(t, n, p, "http://") || lit_at(t, n, p, "ftp://") ||
|
||
lit_at(t, n, p, "file://")) {
|
||
int q = p;
|
||
while (t[q] != '/') {
|
||
q++;
|
||
}
|
||
q += 2;
|
||
if (q < n && !pdf_u_space(t[q])) {
|
||
return run_end_from(t, n, q);
|
||
}
|
||
}
|
||
if (lit_at(t, n, p, "www.") && p + 4 < n && !pdf_u_space(t[p + 4])) {
|
||
return run_end_from(t, n, p + 4);
|
||
}
|
||
if (pdf_u_space(t[p])) {
|
||
return -1;
|
||
}
|
||
url_cache_run(t, n, p, uc);
|
||
if (uc->at > p) {
|
||
return uc->at_end;
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
/* PATH_RE's body after the optional prefix: (?:[seg]+/)+[tail]*[end] */
|
||
static int path_body(const uint32_t *t, int n, int q) {
|
||
int p = q;
|
||
int last_ts = -1;
|
||
for (;;) {
|
||
int r = p;
|
||
while (r < n && path_seg(t[r])) {
|
||
r++;
|
||
}
|
||
if (r > p && r < n && t[r] == '/') {
|
||
last_ts = r + 1;
|
||
p = r + 1;
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
if (last_ts < 0) {
|
||
return -1;
|
||
}
|
||
/* tail starts, from the last segment back to the first */
|
||
int ts = last_ts;
|
||
for (;;) {
|
||
int r = ts;
|
||
int last_e = -1;
|
||
while (r < n && path_tail(t[r])) {
|
||
if (path_end(t[r])) {
|
||
last_e = r;
|
||
}
|
||
r++;
|
||
}
|
||
if (last_e >= 0) {
|
||
return last_e + 1;
|
||
}
|
||
/* give back one segment: the previous '/' before ts-1 */
|
||
int s = ts - 2;
|
||
while (s >= q && t[s] != '/') {
|
||
s--;
|
||
}
|
||
if (s < q) {
|
||
return -1;
|
||
}
|
||
ts = s + 1;
|
||
}
|
||
}
|
||
|
||
static int path_at(const uint32_t *t, int n, int p) {
|
||
if (p > 0) {
|
||
uint32_t b = t[p - 1];
|
||
if (pdf_u_word(b) || b == '/' || b == '.' || b == '-' || b == '~' || b == '$' || b == '@') {
|
||
return -1;
|
||
}
|
||
}
|
||
/* (?:\.{1,2}/|~/|/)? tried in order, then without */
|
||
int e;
|
||
if (p + 2 < n && t[p] == '.' && t[p + 1] == '.' && t[p + 2] == '/' &&
|
||
(e = path_body(t, n, p + 3)) >= 0) {
|
||
return e;
|
||
}
|
||
if (p + 1 < n && t[p] == '.' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) {
|
||
return e;
|
||
}
|
||
if (p + 1 < n && t[p] == '~' && t[p + 1] == '/' && (e = path_body(t, n, p + 2)) >= 0) {
|
||
return e;
|
||
}
|
||
if (t[p] == '/' && (e = path_body(t, n, p + 1)) >= 0) {
|
||
return e;
|
||
}
|
||
return path_body(t, n, p);
|
||
}
|
||
|
||
/* QUAL_RE: [A-Za-z_$][\w$]*(?:(?:\.|::|->|#)[A-Za-z_$][\w$]*)+(?:\(\))? */
|
||
static int qual_at(const uint32_t *t, int n, int p) {
|
||
if (p > 0) {
|
||
uint32_t b = t[p - 1];
|
||
if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') {
|
||
return -1;
|
||
}
|
||
}
|
||
if (!ident_start(t[p])) {
|
||
return -1;
|
||
}
|
||
int q = p + 1;
|
||
while (q < n && ident_cont(t[q])) {
|
||
q++;
|
||
}
|
||
int groups = 0;
|
||
for (;;) {
|
||
int sep = 0;
|
||
if (q < n && t[q] == '.') {
|
||
sep = 1;
|
||
} else if (q + 1 < n && t[q] == ':' && t[q + 1] == ':') {
|
||
sep = 2;
|
||
} else if (q + 1 < n && t[q] == '-' && t[q + 1] == '>') {
|
||
sep = 2;
|
||
} else if (q < n && t[q] == '#') {
|
||
sep = 1;
|
||
}
|
||
if (!sep || q + sep >= n || !ident_start(t[q + sep])) {
|
||
break;
|
||
}
|
||
q += sep + 1;
|
||
while (q < n && ident_cont(t[q])) {
|
||
q++;
|
||
}
|
||
groups++;
|
||
}
|
||
if (!groups) {
|
||
return -1;
|
||
}
|
||
if (q + 1 < n && t[q] == '(' && t[q + 1] == ')') {
|
||
q += 2;
|
||
}
|
||
return q;
|
||
}
|
||
|
||
/* TOK_RE: [A-Za-z_$][\w$]* (whole identifier) */
|
||
static int tok_at(const uint32_t *t, int n, int p) {
|
||
if (p > 0) {
|
||
uint32_t b = t[p - 1];
|
||
if (pdf_u_word(b) || b == '.' || b == '$' || b == '@' || b == '/' || b == '-') {
|
||
return -1;
|
||
}
|
||
}
|
||
if (!ident_start(t[p])) {
|
||
return -1;
|
||
}
|
||
int q = p + 1;
|
||
while (q < n && ident_cont(t[q])) {
|
||
q++;
|
||
}
|
||
return q;
|
||
}
|
||
|
||
static uint32_t ascii_lower(uint32_t c) {
|
||
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
|
||
}
|
||
|
||
static bool in_list_lower(const uint32_t *s, int n, const char *const *list) {
|
||
for (int i = 0; list[i]; i++) {
|
||
const char *w = list[i];
|
||
int k = 0;
|
||
while (k < n && w[k] && ascii_lower(s[k]) == (uint32_t)(unsigned char)w[k]) {
|
||
k++;
|
||
}
|
||
if (k == n && !w[k]) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/* str.islower / str.isupper of a whole string */
|
||
static bool str_islower(const uint32_t *s, int n) {
|
||
bool cased = false;
|
||
for (int i = 0; i < n; i++) {
|
||
if (pdf_u_upper(s[i]) || pdf_u_title(s[i])) {
|
||
return false;
|
||
}
|
||
cased = cased || pdf_u_lower(s[i]);
|
||
}
|
||
return cased;
|
||
}
|
||
|
||
static bool str_isupper(const uint32_t *s, int n) {
|
||
bool cased = false;
|
||
for (int i = 0; i < n; i++) {
|
||
if (pdf_u_lower(s[i]) || pdf_u_title(s[i])) {
|
||
return false;
|
||
}
|
||
cased = cased || pdf_u_upper(s[i]);
|
||
}
|
||
return cased;
|
||
}
|
||
|
||
static bool is_camel(const uint32_t *t, int n) {
|
||
if (n < 3 || str_isupper(t, n) || str_islower(t, n)) {
|
||
return false;
|
||
}
|
||
/* [A-Z]{2,}s: PDFs, APIs, IDs */
|
||
if (t[n - 1] == 's') {
|
||
bool caps = true;
|
||
for (int i = 0; i < n - 1; i++) {
|
||
caps = caps && t[i] >= 'A' && t[i] <= 'Z';
|
||
}
|
||
if (caps) {
|
||
return false;
|
||
}
|
||
}
|
||
for (int i = 0; i + 1 < n; i++) {
|
||
uint32_t a = t[i];
|
||
uint32_t b = t[i + 1];
|
||
if (pdf_u_lower(a) && pdf_u_upper(b)) {
|
||
return true;
|
||
}
|
||
if (i >= 1 && pdf_u_upper(t[i - 1]) && pdf_u_upper(a) && pdf_u_lower(b)) {
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/* The extension after the last '.' of a segment, when it is a code extension. */
|
||
static bool has_code_ext(const uint32_t *s, int n) {
|
||
int dot = -1;
|
||
for (int i = 0; i < n; i++) {
|
||
if (s[i] == '.') {
|
||
dot = i;
|
||
}
|
||
}
|
||
return dot >= 0 && in_list_lower(s + dot + 1, n - dot - 1, CODE_EXT);
|
||
}
|
||
|
||
static void path_candidate(const uint32_t *t, int a, int b, candv_t *cv) {
|
||
/* s.strip("./~"), split on '/', drop empties */
|
||
int sa = a;
|
||
int sb = b;
|
||
while (sa < sb && (t[sa] == '.' || t[sa] == '/' || t[sa] == '~')) {
|
||
sa++;
|
||
}
|
||
while (sb > sa && (t[sb - 1] == '.' || t[sb - 1] == '/' || t[sb - 1] == '~')) {
|
||
sb--;
|
||
}
|
||
bool any_seg = false;
|
||
bool all_numeric = true;
|
||
int last_s = -1;
|
||
int last_e = -1;
|
||
for (int i = sa; i <= sb;) {
|
||
int j = i;
|
||
while (j < sb && t[j] != '/') {
|
||
j++;
|
||
}
|
||
if (j > i) {
|
||
any_seg = true;
|
||
for (int k = i; k < j; k++) {
|
||
if (!((t[k] >= '0' && t[k] <= '9') || t[k] == '.' || t[k] == '-')) {
|
||
all_numeric = false;
|
||
}
|
||
}
|
||
last_s = i;
|
||
last_e = j;
|
||
}
|
||
i = j + 1;
|
||
}
|
||
if (!any_seg || all_numeric) {
|
||
return; /* numeric-path */
|
||
}
|
||
if (in_list_lower(t + a, b - a, SLASH_STOP)) {
|
||
return;
|
||
}
|
||
int slashes = 0;
|
||
for (int i = a; i < b; i++) {
|
||
slashes += t[i] == '/';
|
||
}
|
||
bool has_ext = has_code_ext(t + last_s, last_e - last_s);
|
||
bool prefixed =
|
||
lit_at(t, b, a, "./") || lit_at(t, b, a, "../") || lit_at(t, b, a, "~/") || t[a] == '/';
|
||
cand_t c = {a, b, (slashes >= 2 || has_ext || prefixed) ? MK_PATH : MK_PATH1, 0, 0};
|
||
/* norm: leading ../ ./ ~/ / removed (repeatedly), trailing '/' removed */
|
||
int na = a;
|
||
for (;;) {
|
||
if (lit_at(t, b, na, "../")) {
|
||
na += 3;
|
||
} else if (lit_at(t, b, na, "./") || lit_at(t, b, na, "~/")) {
|
||
na += 2;
|
||
} else if (na < b && t[na] == '/') {
|
||
na += 1;
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
int nb = b;
|
||
while (nb > na && t[nb - 1] == '/') {
|
||
nb--;
|
||
}
|
||
c.norm_start = cv->pool.n;
|
||
for (int i = na; i < nb; i++) {
|
||
u32_push(&cv->pool, t[i]);
|
||
}
|
||
c.norm_len = nb - na;
|
||
cand_push(cv, c);
|
||
}
|
||
|
||
static void qual_candidate(const uint32_t *t, int a, int b, candv_t *cv) {
|
||
int ce = b;
|
||
if (b - a >= 2 && t[b - 2] == '(' && t[b - 1] == ')') {
|
||
ce = b - 2;
|
||
}
|
||
bool colons = false;
|
||
bool arrow = false;
|
||
int ns = cv->pool.n;
|
||
for (int i = a; i < ce; i++) {
|
||
if (i + 1 < ce && t[i] == ':' && t[i + 1] == ':') {
|
||
colons = true;
|
||
u32_push(&cv->pool, '.');
|
||
i++;
|
||
} else if (i + 1 < ce && t[i] == '-' && t[i + 1] == '>') {
|
||
arrow = true;
|
||
u32_push(&cv->pool, '.');
|
||
i++;
|
||
} else if (t[i] == '#') {
|
||
u32_push(&cv->pool, '.');
|
||
} else {
|
||
u32_push(&cv->pool, t[i]);
|
||
}
|
||
}
|
||
if (cv->pool.fail) {
|
||
cv->fail = true;
|
||
return;
|
||
}
|
||
const uint32_t *norm = cv->pool.v + ns;
|
||
int nl = cv->pool.n - ns;
|
||
bool short_segs = true;
|
||
bool has_us = false;
|
||
int seg_s = 0;
|
||
int last_s = 0;
|
||
for (int i = 0; i <= nl; i++) {
|
||
if (i == nl || norm[i] == '.') {
|
||
if (i - seg_s > 2) {
|
||
short_segs = false;
|
||
}
|
||
last_s = seg_s;
|
||
seg_s = i + 1;
|
||
} else if (norm[i] == '_') {
|
||
has_us = true;
|
||
}
|
||
}
|
||
bool lower = str_islower(norm, nl);
|
||
if ((short_segs && lower) ||
|
||
(in_list_lower(norm + last_s, nl - last_s, TLDS) && lower && !has_us)) {
|
||
cv->pool.n = ns; /* abbreviation or domain: taken, no candidate */
|
||
return;
|
||
}
|
||
bool file = in_list_lower(norm + last_s, nl - last_s, CODE_EXT) && !colons && !arrow;
|
||
cand_t c = {a, b, file ? MK_FILE : MK_QUAL, ns, nl};
|
||
cand_push(cv, c);
|
||
}
|
||
|
||
static void tok_candidate(const uint32_t *t, int n, int a, int b, uint8_t *taken, candv_t *cv) {
|
||
int len = b - a;
|
||
/* "_" in t.strip("_") */
|
||
int sa = a;
|
||
int sb = b;
|
||
while (sa < sb && t[sa] == '_') {
|
||
sa++;
|
||
}
|
||
while (sb > sa && t[sb - 1] == '_') {
|
||
sb--;
|
||
}
|
||
bool inner_us = false;
|
||
for (int i = sa; i < sb; i++) {
|
||
inner_us = inner_us || t[i] == '_';
|
||
}
|
||
bool dunder = len > 4 && t[a] == '_' && t[a + 1] == '_' && t[b - 1] == '_' && t[b - 2] == '_';
|
||
mention_kind_t kind;
|
||
if (inner_us || dunder) {
|
||
kind = MK_SNAKE;
|
||
} else if (is_camel(t + a, len)) {
|
||
kind = MK_CAMEL;
|
||
} else if (b < n && t[b] == '(' && len >= 2 && !str_isupper(t + a, len)) {
|
||
kind = MK_CALL;
|
||
} else {
|
||
return;
|
||
}
|
||
span_take(taken, a, b);
|
||
cand_t c = {a, b, kind, cv->pool.n, len};
|
||
for (int i = a; i < b; i++) {
|
||
u32_push(&cv->pool, t[i]);
|
||
}
|
||
cand_push(cv, c);
|
||
}
|
||
|
||
static int cand_cmp(const void *x, const void *y) {
|
||
const cand_t *a = (const cand_t *)x;
|
||
const cand_t *b = (const cand_t *)y;
|
||
return a->start < b->start ? -1 : a->start > b->start ? 1 : 0;
|
||
}
|
||
|
||
/* candidates(): line-local candidates of text, sorted by start. */
|
||
static bool candidates(const uint32_t *t, int n, bool with_tok, candv_t *cv) {
|
||
uint8_t *taken = (uint8_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(n ? n : 1));
|
||
if (!taken) {
|
||
return false;
|
||
}
|
||
url_cache_t uc = {-1, -1, -1, -1};
|
||
for (int p = 0; p < n;) {
|
||
int e = url_at(t, n, p, &uc);
|
||
if (e > p) {
|
||
span_take(taken, p, e);
|
||
p = e;
|
||
} else {
|
||
p++;
|
||
}
|
||
}
|
||
for (int p = 0; p < n;) {
|
||
int e = path_at(t, n, p);
|
||
if (e <= p) {
|
||
p++;
|
||
continue;
|
||
}
|
||
if (span_free(taken, p, e)) {
|
||
span_take(taken, p, e);
|
||
path_candidate(t, p, e, cv);
|
||
}
|
||
p = e;
|
||
}
|
||
for (int p = 0; p < n;) {
|
||
int e = qual_at(t, n, p);
|
||
if (e <= p) {
|
||
p++;
|
||
continue;
|
||
}
|
||
if (span_free(taken, p, e)) {
|
||
span_take(taken, p, e);
|
||
qual_candidate(t, p, e, cv);
|
||
}
|
||
p = e;
|
||
}
|
||
if (with_tok) {
|
||
for (int p = 0; p < n;) {
|
||
int e = tok_at(t, n, p);
|
||
if (e <= p) {
|
||
p++;
|
||
continue;
|
||
}
|
||
if (span_free(taken, p, e)) {
|
||
tok_candidate(t, n, p, e, taken, cv);
|
||
}
|
||
p = e;
|
||
}
|
||
}
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, taken);
|
||
if (cv->n > 1) {
|
||
qsort(cv->v, (size_t)cv->n, sizeof(cand_t), cand_cmp);
|
||
}
|
||
return !cv->fail && !cv->pool.fail;
|
||
}
|
||
|
||
/* ── Joins across a line break ───────────────────────────────────── */
|
||
|
||
typedef enum { JK_NONE = 0, JK_SOFT, JK_HYPHEN, JK_CODE } join_kind_t;
|
||
|
||
static join_kind_t join_kind(const uint32_t *prev, int pn, const uint32_t *nxt, int nn) {
|
||
if (!pn || !nn) {
|
||
return JK_NONE;
|
||
}
|
||
if (prev[pn - 1] == 0x00AD && pdf_u_alpha(nxt[0])) {
|
||
return JK_SOFT;
|
||
}
|
||
if (prev[pn - 1] == '-' && pn >= 2 && pdf_u_alpha(prev[pn - 2]) && pdf_u_lower(nxt[0])) {
|
||
return JK_HYPHEN;
|
||
}
|
||
static const char *const sufs[] = {"::", "->", "_", "/"};
|
||
for (int k = 0; k < 4; k++) {
|
||
int sl = (int)strlen(sufs[k]);
|
||
if (pn > sl && lit_at(prev, pn, pn - sl, sufs[k]) && idc(prev[pn - sl - 1]) &&
|
||
idc(nxt[0])) {
|
||
return JK_CODE;
|
||
}
|
||
}
|
||
if (idc(prev[pn - 1])) {
|
||
/* (?:\.|::|->|_)[A-Za-z_] | \(\) at the start of nxt */
|
||
int s = 0;
|
||
if (nxt[0] == '.' || nxt[0] == '_') {
|
||
s = 1;
|
||
} else if (nn >= 2 &&
|
||
((nxt[0] == ':' && nxt[1] == ':') || (nxt[0] == '-' && nxt[1] == '>'))) {
|
||
s = 2;
|
||
}
|
||
if (s && s < nn && (is_alpha_ascii(nxt[s]) || nxt[s] == '_')) {
|
||
return JK_CODE;
|
||
}
|
||
if (nn >= 2 && nxt[0] == '(' && nxt[1] == ')') {
|
||
return JK_CODE;
|
||
}
|
||
}
|
||
return JK_NONE;
|
||
}
|
||
|
||
/* [A-Za-z0-9_$@.:/\-#>] and [A-Za-z0-9_$@.:/\-#>()] */
|
||
static bool left_run_char(uint32_t c) {
|
||
return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' ||
|
||
c == '>' || c == 0x00AD;
|
||
}
|
||
|
||
static bool right_run_char(uint32_t c) {
|
||
return idc(c) || c == '@' || c == '.' || c == ':' || c == '/' || c == '-' || c == '#' ||
|
||
c == '>' || c == '(' || c == ')';
|
||
}
|
||
|
||
/* ── Emission ────────────────────────────────────────────────────── */
|
||
|
||
typedef struct {
|
||
CBMExtractCtx *ctx;
|
||
const char *page_qn;
|
||
uint32_t page;
|
||
const uint32_t *flat;
|
||
int n;
|
||
const int *line_starts;
|
||
int nlines;
|
||
bool failed;
|
||
} pdf_emit_t;
|
||
|
||
static uint32_t line_of(const pdf_emit_t *e, int off) {
|
||
int lo = 0;
|
||
int hi = e->nlines - 1;
|
||
while (lo < hi) {
|
||
int mid = lo + (hi - lo + 1) / 2;
|
||
if (e->line_starts[mid] <= off) {
|
||
lo = mid;
|
||
} else {
|
||
hi = mid - 1;
|
||
}
|
||
}
|
||
return (uint32_t)lo + 1;
|
||
}
|
||
|
||
/* R3: "foo.bar=" right after "(" or "," -- a keyword argument, not a reference */
|
||
static bool kwarg_at(const pdf_emit_t *e, int start, int end) {
|
||
if (end >= e->n || e->flat[end] != '=' || (end + 1 < e->n && e->flat[end + 1] == '=')) {
|
||
return false;
|
||
}
|
||
int lim = start - PDF_R3_WINDOW < 0 ? 0 : start - PDF_R3_WINDOW;
|
||
for (int i = start - 1; i >= lim; i--) {
|
||
if (!pdf_u_space(e->flat[i])) {
|
||
return e->flat[i] == '(' || e->flat[i] == ',';
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static int u8_put(uint32_t cp, char *o) {
|
||
if (cp < 0x80) {
|
||
o[0] = (char)cp;
|
||
return 1;
|
||
}
|
||
if (cp < 0x800) {
|
||
o[0] = (char)(0xC0 | (cp >> 6));
|
||
o[1] = (char)(0x80 | (cp & 0x3F));
|
||
return 2;
|
||
}
|
||
if (cp < 0x10000) {
|
||
o[0] = (char)(0xE0 | (cp >> 12));
|
||
o[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||
o[2] = (char)(0x80 | (cp & 0x3F));
|
||
return 3;
|
||
}
|
||
o[0] = (char)(0xF0 | (cp >> 18));
|
||
o[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
|
||
o[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||
o[3] = (char)(0x80 | (cp & 0x3F));
|
||
return 4;
|
||
}
|
||
|
||
static int kind_syntax(mention_kind_t k) {
|
||
switch (k) {
|
||
case MK_PATH:
|
||
case MK_PATH1:
|
||
return CBM_DOCLINK_PDF_PATH;
|
||
case MK_FILE:
|
||
return CBM_DOCLINK_PDF_FILE;
|
||
case MK_QUAL:
|
||
return CBM_DOCLINK_PDF_QN;
|
||
default:
|
||
return CBM_DOCLINK_PDF_NAME;
|
||
}
|
||
}
|
||
|
||
static bool structural(mention_kind_t k) {
|
||
return k == MK_PATH || k == MK_PATH1 || k == MK_FILE || k == MK_QUAL;
|
||
}
|
||
|
||
/* Push a token; returns its index in the file's token array, or -1. */
|
||
static int emit(pdf_emit_t *e, int syntax, const char *raw, uint32_t line, uint16_t flags) {
|
||
CBMArena *a = e->ctx->arena;
|
||
char *r = cbm_arena_strdup(a, raw);
|
||
if (!r) {
|
||
e->failed = true;
|
||
return -1;
|
||
}
|
||
CBMDocLink link = {
|
||
.source_qn = e->page_qn,
|
||
.raw = r,
|
||
.line = line,
|
||
.def_line = e->page,
|
||
.syntax = (uint16_t)syntax,
|
||
.flags = flags,
|
||
};
|
||
int idx = e->ctx->result->doc_links.count;
|
||
cbm_doclinks_push(&e->ctx->result->doc_links, a, link);
|
||
if (e->ctx->result->doc_links.count != idx + 1) {
|
||
e->failed = true;
|
||
return -1;
|
||
}
|
||
return idx;
|
||
}
|
||
|
||
typedef struct {
|
||
char *p;
|
||
size_t n;
|
||
size_t cap;
|
||
bool fail;
|
||
} sbuf_t;
|
||
|
||
static void sb_put(sbuf_t *b, const char *s, size_t n) {
|
||
if (b->fail) {
|
||
return;
|
||
}
|
||
if (b->n + n + 1 > b->cap) {
|
||
size_t ncap = b->cap ? b->cap * 2 : 256;
|
||
while (ncap < b->n + n + 1) {
|
||
ncap *= 2;
|
||
}
|
||
char *g = (char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, b->p, ncap);
|
||
if (!g) {
|
||
b->fail = true;
|
||
return;
|
||
}
|
||
b->p = g;
|
||
b->cap = ncap;
|
||
}
|
||
memcpy(b->p + b->n, s, n);
|
||
b->n += n;
|
||
b->p[b->n] = '\0';
|
||
}
|
||
|
||
static void sb_cps(sbuf_t *b, const uint32_t *s, int n) {
|
||
char u[PDF_U8_MAX];
|
||
for (int i = 0; i < n; i++) {
|
||
int k = u8_put(s[i], u);
|
||
sb_put(b, u, (size_t)k);
|
||
}
|
||
}
|
||
|
||
/* One page: its line-local structural tokens, then its join proposals. */
|
||
static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page,
|
||
const char *page_qn) {
|
||
u32v_t norm = {0};
|
||
if (!normalise(text, len, &norm)) {
|
||
u32_free(&norm);
|
||
ctx->result->doc_links.failed = true;
|
||
return;
|
||
}
|
||
/* lines, each stripped; flat = lines joined by '\n' */
|
||
u32v_t flat = {0};
|
||
int *starts = NULL;
|
||
int nlines = 0;
|
||
int cap = 0;
|
||
int *lens = NULL;
|
||
for (int i = 0; i <= norm.n;) {
|
||
int j = i;
|
||
while (j < norm.n && norm.v[j] != '\n') {
|
||
j++;
|
||
}
|
||
int a = i;
|
||
int b = j;
|
||
while (a < b && pdf_u_space(norm.v[a])) {
|
||
a++;
|
||
}
|
||
while (b > a && pdf_u_space(norm.v[b - 1])) {
|
||
b--;
|
||
}
|
||
if (nlines == cap) {
|
||
cap = cap ? cap * 2 : PDF_LIST_MIN;
|
||
int *g1 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, starts, (size_t)cap * sizeof(int));
|
||
if (g1) {
|
||
starts = g1;
|
||
}
|
||
int *g2 = (int *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, lens, (size_t)cap * sizeof(int));
|
||
if (g2) {
|
||
lens = g2;
|
||
}
|
||
if (!g1 || !g2) {
|
||
flat.fail = true;
|
||
break;
|
||
}
|
||
}
|
||
if (nlines) {
|
||
u32_push(&flat, '\n');
|
||
}
|
||
starts[nlines] = flat.n;
|
||
lens[nlines] = b - a;
|
||
nlines++;
|
||
for (int k = a; k < b; k++) {
|
||
u32_push(&flat, norm.v[k]);
|
||
}
|
||
if (j >= norm.n) {
|
||
break;
|
||
}
|
||
i = j + 1;
|
||
}
|
||
u32_free(&norm);
|
||
candv_t cv = {0};
|
||
if (flat.fail || !candidates(flat.v, flat.n, false, &cv)) {
|
||
ctx->result->doc_links.failed = true;
|
||
goto done;
|
||
}
|
||
pdf_emit_t e = {ctx, page_qn, page, flat.v, flat.n, starts, nlines, false};
|
||
/* the token index of every structural candidate (-1: not emitted) */
|
||
int *tok = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)(cv.n ? cv.n : 1) * sizeof(int));
|
||
if (!tok) {
|
||
ctx->result->doc_links.failed = true;
|
||
goto done;
|
||
}
|
||
sbuf_t sb = {0};
|
||
for (int i = 0; i < cv.n; i++) {
|
||
const cand_t *c = &cv.v[i];
|
||
tok[i] = -1;
|
||
if (!structural(c->kind) || kwarg_at(&e, c->start, c->end)) {
|
||
continue;
|
||
}
|
||
sb.n = 0;
|
||
sb_cps(&sb, cv.pool.v + c->norm_start, c->norm_len);
|
||
if (sb.fail) {
|
||
break;
|
||
}
|
||
tok[i] = emit(&e, kind_syntax(c->kind), sb.p ? sb.p : "", line_of(&e, c->start), 0);
|
||
}
|
||
/* The emitted candidates of each line, ascending (CSR: lc[lc_off[l] ..
|
||
* lc_off[l + 1])). A join's span covers the end of one line and the start of
|
||
* the next, so only their candidates can overlap it; reading every candidate
|
||
* of the page per join cost (joins x candidates), quadratic in the text. */
|
||
int *lc_off = (int *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(nlines + 2) * sizeof(int));
|
||
int *lc = NULL;
|
||
if (!lc_off) {
|
||
e.failed = true;
|
||
} else {
|
||
for (int f = 0; f < cv.n; f++) {
|
||
if (tok[f] >= 0) {
|
||
int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;
|
||
for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {
|
||
lc_off[l]++; /* line_of is 1-based: line l - 1 counts at l */
|
||
}
|
||
}
|
||
}
|
||
for (int l = 0; l < nlines; l++) {
|
||
lc_off[l + 1] += lc_off[l];
|
||
}
|
||
lc = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT,
|
||
(size_t)(lc_off[nlines] ? lc_off[nlines] : 1) * sizeof(int));
|
||
if (!lc) {
|
||
e.failed = true;
|
||
} else {
|
||
/* lc_off[l] runs as line l's cursor, then shifts back to its start */
|
||
for (int f = 0; f < cv.n; f++) {
|
||
if (tok[f] >= 0) {
|
||
int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;
|
||
for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {
|
||
lc[lc_off[l - 1]++] = f;
|
||
}
|
||
}
|
||
}
|
||
for (int l = nlines; l > 0; l--) {
|
||
lc_off[l] = lc_off[l - 1];
|
||
}
|
||
lc_off[0] = 0;
|
||
}
|
||
}
|
||
/* joins */
|
||
for (int li = 0; li + 1 < nlines && !sb.fail && !e.failed; li++) {
|
||
const uint32_t *prev = flat.v + starts[li];
|
||
const uint32_t *nxt = flat.v + starts[li + 1];
|
||
int pn = lens[li];
|
||
int nn = lens[li + 1];
|
||
join_kind_t jk = join_kind(prev, pn, nxt, nn);
|
||
if (jk == JK_NONE) {
|
||
continue;
|
||
}
|
||
int ls = pn;
|
||
while (ls > 0 && left_run_char(prev[ls - 1])) {
|
||
ls--;
|
||
}
|
||
int re = 0;
|
||
while (re < nn && right_run_char(nxt[re])) {
|
||
re++;
|
||
}
|
||
if (ls == pn || re == 0) {
|
||
continue;
|
||
}
|
||
int left_len = pn - ls;
|
||
int dropped = (jk == JK_HYPHEN || jk == JK_SOFT) ? 1 : 0;
|
||
int junction = left_len - dropped;
|
||
u32v_t joined = {0};
|
||
for (int k = ls; k < pn - dropped; k++) {
|
||
u32_push(&joined, prev[k]);
|
||
}
|
||
for (int k = 0; k < re; k++) {
|
||
u32_push(&joined, nxt[k]);
|
||
}
|
||
candv_t jc = {0};
|
||
if (joined.fail || !candidates(joined.v, joined.n, true, &jc)) {
|
||
u32_free(&joined);
|
||
candv_free(&jc);
|
||
ctx->result->doc_links.failed = true;
|
||
break;
|
||
}
|
||
int a0 = starts[li] + ls;
|
||
int b0 = starts[li + 1];
|
||
for (int k = 0; k < jc.n; k++) {
|
||
const cand_t *c = &jc.v[k];
|
||
if (!(c->start < junction && junction < c->end)) {
|
||
continue;
|
||
}
|
||
int s_flat = a0 + c->start;
|
||
int e_flat = b0 + (c->end - junction);
|
||
if (structural(c->kind) && kwarg_at(&e, s_flat, e_flat)) {
|
||
continue;
|
||
}
|
||
sb.n = 0;
|
||
const uint32_t *nm = jc.pool.v + c->norm_start;
|
||
sb_cps(&sb, nm, c->norm_len);
|
||
if (dropped) {
|
||
/* alts: the norm with the hyphen kept, at the junction */
|
||
/* Python slices clamp: past the end, the hyphen goes last */
|
||
int at = junction - c->start;
|
||
at = at > c->norm_len ? c->norm_len : at;
|
||
char sep = PDF_JOIN_SEP;
|
||
sb_put(&sb, &sep, 1);
|
||
sb_cps(&sb, nm, at);
|
||
sb_put(&sb, "-", 1);
|
||
sb_cps(&sb, nm + at, c->norm_len - at);
|
||
}
|
||
char fs = PDF_JOIN_FRAG;
|
||
sb_put(&sb, &fs, 1);
|
||
bool first = true;
|
||
int ia = lc_off[li];
|
||
int ea = lc_off[li + 1];
|
||
int ib = lc_off[li + 1];
|
||
int eb = lc_off[li + 2];
|
||
while (ia < ea || ib < eb) {
|
||
int f;
|
||
if (ib >= eb || (ia < ea && lc[ia] < lc[ib])) {
|
||
f = lc[ia++];
|
||
} else if (ia >= ea || lc[ib] < lc[ia]) {
|
||
f = lc[ib++];
|
||
} else {
|
||
f = lc[ia++]; /* on both lines */
|
||
ib++;
|
||
}
|
||
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
|
||
atomic_fetch_add_explicit(&g_join_steps, 1, memory_order_relaxed);
|
||
#endif
|
||
const cand_t *x = &cv.v[f];
|
||
if (x->start < e_flat && x->end > s_flat) {
|
||
char num[16];
|
||
int nlen = snprintf(num, sizeof(num), first ? "%d" : ",%d", tok[f]);
|
||
sb_put(&sb, num, (size_t)nlen);
|
||
first = false;
|
||
}
|
||
}
|
||
if (sb.fail) {
|
||
break;
|
||
}
|
||
emit(&e, kind_syntax(c->kind), sb.p, line_of(&e, s_flat), CBM_DOCLINK_FLAG_JOIN);
|
||
}
|
||
u32_free(&joined);
|
||
candv_free(&jc);
|
||
}
|
||
if (sb.fail || e.failed) {
|
||
ctx->result->doc_links.failed = true;
|
||
}
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, sb.p);
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, tok);
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, lc_off);
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, lc);
|
||
done:
|
||
candv_free(&cv);
|
||
u32_free(&flat);
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, starts);
|
||
cbm_free(CBM_MEM_CLASS_EXTRACT, lens);
|
||
}
|
||
|
||
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
|
||
void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page,
|
||
const char *page_qn) {
|
||
scan_page(ctx, text, len, page, page_qn);
|
||
}
|
||
|
||
size_t cbm_pdf_test_join_steps(void) {
|
||
return atomic_load_explicit(&g_join_steps, memory_order_relaxed);
|
||
}
|
||
#endif
|
||
|
||
/* ── The document ────────────────────────────────────────────────── */
|
||
|
||
/* R1: the field test's category(): a PDF under a test or fixture directory. */
|
||
static bool fixture_pdf(const char *rel_path) {
|
||
const char *s = rel_path;
|
||
for (;;) {
|
||
const char *slash = strchr(s, '/');
|
||
if (!slash) {
|
||
return false;
|
||
}
|
||
size_t n = (size_t)(slash - s);
|
||
char comp[256];
|
||
if (n < sizeof(comp)) {
|
||
for (size_t i = 0; i < n; i++) {
|
||
comp[i] = (char)(s[i] >= 'A' && s[i] <= 'Z' ? s[i] + ('a' - 'A') : s[i]);
|
||
}
|
||
comp[n] = '\0';
|
||
for (int i = 0; FIXTURE_DIRS[i]; i++) {
|
||
if (strcmp(comp, FIXTURE_DIRS[i]) == 0) {
|
||
return true;
|
||
}
|
||
}
|
||
if (strstr(comp, "test") && strcmp(comp, "testing") != 0) {
|
||
return true;
|
||
}
|
||
}
|
||
s = slash + 1;
|
||
}
|
||
}
|
||
|
||
/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes
|
||
* cut on a character boundary and then said so. A page holds a few kilobytes;
|
||
* only a crafted text layer reaches the bound, and the mentions are still read
|
||
* from the whole text. */
|
||
static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) {
|
||
if (pg->len <= PDF_PAGE_DOC_MAX) {
|
||
return pg->text;
|
||
}
|
||
size_t cut = PDF_PAGE_DOC_MAX;
|
||
while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) {
|
||
cut--; /* not inside a UTF-8 sequence */
|
||
}
|
||
const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut,
|
||
pg->text, cut, pg->len);
|
||
return doc ? doc : pg->text;
|
||
}
|
||
|
||
void cbm_pdf_extract_document(CBMExtractCtx *ctx) {
|
||
CBMFileResult *result = ctx->result;
|
||
CBMArena *a = ctx->arena;
|
||
cbm_pdf_result_t pr;
|
||
if (!cbm_pdf_extract((const unsigned char *)ctx->source, (size_t)ctx->source_len, a, &pr)) {
|
||
result->has_error = true;
|
||
result->error_msg = cbm_arena_strdup(a, "pdf: out of memory");
|
||
return;
|
||
}
|
||
if (pr.status != CBM_PDF_OK) {
|
||
static const char *const why[] = {"ok", "pdf: not a PDF", "pdf: no pages", "pdf: encrypted",
|
||
"pdf: out of memory"};
|
||
result->has_error = pr.status != CBM_PDF_ENCRYPTED;
|
||
result->error_msg = cbm_arena_strdup(a, why[pr.status]);
|
||
return;
|
||
}
|
||
bool fixture = fixture_pdf(ctx->rel_path ? ctx->rel_path : "");
|
||
for (int i = 0; i < pr.npages; i++) {
|
||
uint32_t page = (uint32_t)i + 1;
|
||
char name[PDF_PAGE_NAME];
|
||
char seg[PDF_PAGE_NAME];
|
||
snprintf(name, sizeof(name), "page %u", page);
|
||
snprintf(seg, sizeof(seg), "page_%u", page);
|
||
char *nm = cbm_arena_strdup(a, name);
|
||
char *qn = cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg);
|
||
char *pv = cbm_arena_sprintf(a, "%u", page);
|
||
const char **kv = (const char **)cbm_arena_alloc(a, 3 * sizeof(char *));
|
||
if (!nm || !qn || !pv || !kv) {
|
||
result->has_error = true;
|
||
return;
|
||
}
|
||
kv[0] = "page";
|
||
kv[1] = pv;
|
||
kv[2] = NULL;
|
||
CBMDefinition def;
|
||
memset(&def, 0, sizeof(def));
|
||
def.name = nm;
|
||
def.qualified_name = qn;
|
||
def.label = "Section";
|
||
def.file_path = ctx->rel_path;
|
||
def.start_line = page;
|
||
def.end_line = page;
|
||
def.is_exported = true;
|
||
def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL;
|
||
def.extra_props = kv;
|
||
cbm_defs_push(&result->defs, a, def);
|
||
if (!fixture && pr.pages[i].len) {
|
||
scan_page(ctx, pr.pages[i].text, pr.pages[i].len, page, qn);
|
||
}
|
||
}
|
||
}
|