mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
Martin Vogel
033ca372a8
A PDF is now indexed: its text layer is read by a dependency-free extractor
(internal/cbm/pdf, zlib only) and every page becomes a Section node ("page N",
the page's text as its docstring, property page), so search finds what a page
says and get_code_snippet shows a page's text, never the file's bytes. The code
a page names structurally becomes MENTIONS edges from that page, via "pdf":
pdf_path a repository path ("pkg/server/handler.go", "./x/y")
pdf_file a file name with a code extension ("models.py", "Pool.sol")
pdf_qn a qualified name ("app.models.User.save", "a::b", "x->y")
Bare identifiers never become edges (field test E6c: a type-target rule failed
its pre-registered bar), and only EXACT or UNIQUE resolutions link: the field
test's tiers (path-exact, component-aligned path-suffix, qn-exact, qn-suffix,
the latter also through a node's owner -- a Go method's QN has no receiver --
and with the same one-entity collapses). A mention split across a line break is
proposed as one reference and replaces its fragments only when it resolves.
Hygiene as measured: no links from PDFs under test or fixture directories (R1),
none to data files (R2), none for keyword arguments (R3); a test, mock or
fixture target is a test_only_target row (R4, with the test directories the
field test found missing). A path or name whose first component is not of this
repository is no row at all.
The extractor is a port of the field-tested prototype: xref tables and
streams, /Prev chains, repair near a wrong offset and reconstruction from the
object headers, object streams, Flate (with partial recovery), LZW, ASCII85,
ASCIIHex, RunLength and PNG/TIFF predictors, simple fonts (base encodings,
/Differences, built-in Type1 encodings, standard-14 widths, Type3), ToUnicode
and encoding CMaps, Type0/CID fonts with the embedded TrueType cmap read
backwards, Form XObjects, inline images skipped, word and line breaks from
glyph geometry, right-to-left runs reversed. Its geometry is bit-exact on every
platform (no FMA contraction; CPython's hypot). Untrusted input: every read is
bounds-checked, recursion is depth-limited, integers from the file saturate, a
decoded stream is cut at 512 MiB, and every lookup that could rescan the file
per object or per glyph is answered from an index built once. Not read:
encrypted documents (reported as such); scanned pages have no text layer.
The mention scanner is the field test's (idents.py) with CPython's semantics:
NFKC per code point plus canonical composition and the str predicates, from
generated Unicode 16.0 tables (pdf_unicode.c), and re's leftmost-first
matching.
Pipeline: CBM_LANG_PDF (no grammar; the registry ledger gets a DOCUMENT
capability, the call-node manifest exempts it like the other grammar-free
classification), extraction before any grammar lookup, a resolver hook
file_prepare (a file's state from all its references before any resolves).
Incremental: a proxy node carries parent_class where owner + name differs
from its QN (Go methods, class-body variables), so a re-resolved document binds
a Go method as a full build does.
Proof against the field-test prototype on its corpus (749 PDFs from 21
repositories, restored at the recorded commits, sha256-verified): text
byte-identical on all 1,492 pages of the 720 comparable PDFs (0.73 s vs 14.7 s);
mention tokens identical on all 960 page-text sets of E6, E6b and E6c (26,435
tokens); links identical to the frozen Python pipeline run over the same graph
on semantic-kernel 5/5, cosmos-sdk 30/30, Polly 11/11, mediastore3 3/3, dask
4/4. 44,070 seeded mutants (truncations, flips, nesting and keyword bombs, on
the originals and on uncompressed copies) under ASan and UBSan: 0 findings
after the first campaign's integer overflows were fixed as a class.
Tests: suite doc_links_pdf (layout, fonts and CMaps, object and xref streams,
xref repair and reconstruction, Form XObjects incl. a self-referencing one,
inline images, filters, hostile input, the scanner against the field test's
output, the pipeline's tiers, hygiene and rows, back navigation, the snippet,
incremental == full).
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
104 lines
3.0 KiB
C
104 lines
3.0 KiB
C
/*
|
|
* pdf_unicode.h — the Unicode facts the PDF mention scanner needs, exactly
|
|
* as CPython answers them (pdf_unicode.c is generated from CPython's
|
|
* unicodedata; the field-tested scanner ran on CPython's str and re).
|
|
*/
|
|
#ifndef CBM_PDF_UNICODE_H
|
|
#define CBM_PDF_UNICODE_H
|
|
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
|
|
typedef struct {
|
|
uint32_t cp;
|
|
uint16_t off; /* into PDF_NFKC_SEQ */
|
|
uint8_t len;
|
|
} pdf_nfkc_t;
|
|
|
|
typedef struct {
|
|
uint32_t first;
|
|
uint32_t second;
|
|
uint32_t composite;
|
|
} pdf_ucomp_t;
|
|
|
|
typedef struct {
|
|
uint32_t lo;
|
|
uint32_t hi;
|
|
} pdf_urange_t;
|
|
|
|
extern const pdf_nfkc_t PDF_NFKC[]; /* sorted by cp: NFKC of one code point where it differs */
|
|
extern const int PDF_NFKC_COUNT;
|
|
extern const uint32_t PDF_NFKC_SEQ[];
|
|
extern const pdf_ucomp_t PDF_UCOMP[]; /* sorted by (first, second): canonical compositions */
|
|
extern const int PDF_UCOMP_COUNT;
|
|
extern const pdf_urange_t PDF_U_SPACE[]; /* str.isspace */
|
|
extern const int PDF_U_SPACE_COUNT;
|
|
extern const pdf_urange_t PDF_U_WORD[]; /* re's \w: str.isalnum or '_' */
|
|
extern const int PDF_U_WORD_COUNT;
|
|
extern const pdf_urange_t PDF_U_ALPHA[]; /* str.isalpha */
|
|
extern const int PDF_U_ALPHA_COUNT;
|
|
extern const pdf_urange_t PDF_U_UPPER[]; /* one-character str.isupper */
|
|
extern const int PDF_U_UPPER_COUNT;
|
|
extern const pdf_urange_t PDF_U_LOWER[]; /* one-character str.islower */
|
|
extern const int PDF_U_LOWER_COUNT;
|
|
extern const pdf_urange_t PDF_U_TITLE[]; /* titlecase letters (neither upper nor lower) */
|
|
extern const int PDF_U_TITLE_COUNT;
|
|
|
|
static inline bool pdf_u_in(const pdf_urange_t *r, int n, uint32_t cp) {
|
|
int lo = 0;
|
|
int hi = n - 1;
|
|
while (lo <= hi) {
|
|
int mid = lo + (hi - lo) / 2;
|
|
if (r[mid].hi < cp) {
|
|
lo = mid + 1;
|
|
} else if (r[mid].lo > cp) {
|
|
hi = mid - 1;
|
|
} else {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static inline bool pdf_u_space(uint32_t c) {
|
|
if (c < 0x80) {
|
|
return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F);
|
|
}
|
|
return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, c);
|
|
}
|
|
|
|
static inline bool pdf_u_word(uint32_t c) {
|
|
if (c < 0x80) {
|
|
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
|
c == '_';
|
|
}
|
|
return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, c);
|
|
}
|
|
|
|
static inline bool pdf_u_alpha(uint32_t c) {
|
|
if (c < 0x80) {
|
|
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
|
}
|
|
return pdf_u_in(PDF_U_ALPHA, PDF_U_ALPHA_COUNT, c);
|
|
}
|
|
|
|
static inline bool pdf_u_upper(uint32_t c) {
|
|
if (c < 0x80) {
|
|
return c >= 'A' && c <= 'Z';
|
|
}
|
|
return pdf_u_in(PDF_U_UPPER, PDF_U_UPPER_COUNT, c);
|
|
}
|
|
|
|
static inline bool pdf_u_lower(uint32_t c) {
|
|
if (c < 0x80) {
|
|
return c >= 'a' && c <= 'z';
|
|
}
|
|
return pdf_u_in(PDF_U_LOWER, PDF_U_LOWER_COUNT, c);
|
|
}
|
|
|
|
static inline bool pdf_u_title(uint32_t c) {
|
|
return c >= 0x80 && pdf_u_in(PDF_U_TITLE, PDF_U_TITLE_COUNT, c);
|
|
}
|
|
|
|
#endif /* CBM_PDF_UNICODE_H */
|