From f403f921dd41fd6287c0c9f3303a29e5fb7cfcfb Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 03:44:00 +0200 Subject: [PATCH] pdf: a page's docstring keeps 256 KiB of its text and says so A PDF page's Section carried the whole page text as its docstring, which is also what get_code_snippet returns for it: a crafted text layer stored megabytes per node and handed them to an agent whole (D6 of the document layer security review). Markdown, reStructuredText and AsciiDoc sections keep a short docstring already. A page's docstring is now its text up to 256 KiB, cut on a character boundary and followed by a line that says where it was cut and how long the text is. Pages hold a few kilobytes; the mentions are still read from the whole text. Test: pdf_page_doc_bounded: a 315 KB page keeps 256 KiB plus the note, and a path written at its end is still a mention (the whole text is stored with the bound reverted). Signed-off-by: Martin Vogel --- internal/cbm/doc_pdf.c | 20 +++++++++++++++++- tests/test_doc_links_pdf.c | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.c index 586417b2e..b3ca4a4f3 100644 --- a/internal/cbm/doc_pdf.c +++ b/internal/cbm/doc_pdf.c @@ -45,6 +45,7 @@ enum { PDF_PAGE_NAME = 32, PDF_LIST_MIN = 64, PDF_U8_MAX = 4, + PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */ }; #define PDF_JOIN_SEP '\x1f' /* between a join's forms */ @@ -1321,6 +1322,23 @@ static bool fixture_pdf(const char *rel_path) { } } +/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes + * cut on a character boundary and then said so. A page holds a few kilobytes; + * only a crafted text layer reaches the bound, and the mentions are still read + * from the whole text. */ +static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) { + if (pg->len <= PDF_PAGE_DOC_MAX) { + return pg->text; + } + size_t cut = PDF_PAGE_DOC_MAX; + while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) { + cut--; /* not inside a UTF-8 sequence */ + } + const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut, + pg->text, cut, pg->len); + return doc ? doc : pg->text; +} + void cbm_pdf_extract_document(CBMExtractCtx *ctx) { CBMFileResult *result = ctx->result; CBMArena *a = ctx->arena; @@ -1364,7 +1382,7 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx) { def.start_line = page; def.end_line = page; def.is_exported = true; - def.docstring = pr.pages[i].len ? pr.pages[i].text : NULL; + def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL; def.extra_props = kv; cbm_defs_push(&result->defs, a, def); if (!fixture && pr.pages[i].len) { diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c index ee4e77996..ff96cec01 100644 --- a/tests/test_doc_links_pdf.c +++ b/tests/test_doc_links_pdf.c @@ -1133,6 +1133,48 @@ TEST(pdf_links_pipeline) { PASS(); } +/* A page's docstring keeps 256 KiB of its text and says where it was cut; a + * page's mentions still come from the whole text. */ +TEST(pdf_page_doc_bounded) { + enum { WORDS = 45000 }; /* 315 KB of text */ + tp_buf_t c = {0}; + tp_puts(&c, "BT /F1 12 Tf 72 700 Td ("); + for (int i = 0; i < WORDS; i++) { + tp_puts(&c, "filler "); + } + tp_puts(&c, "src/tail/last.py) Tj ET"); + tp_obj_t o[5] = { + {TP_CATALOG, NULL, 0}, {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0}, + {TP_PAGE_F1_5, NULL, 0}, {TP_HELV, NULL, 0}, + {"", c.p, c.n}, + }; + size_t len; + unsigned char *pdf = tp_pdf(o, 5, NULL, &len); + CBMFileResult *r = + cbm_extract_file((const char *)pdf, (int)len, CBM_LANG_PDF, "p", "doc.pdf", 0, NULL, NULL); + ASSERT_NOT_NULL(r); + const CBMDefinition *page = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Section") == 0) { + page = &r->defs.items[i]; + } + } + ASSERT_NOT_NULL(page); + ASSERT_NOT_NULL(page->docstring); + size_t dl = strlen(page->docstring); + ASSERT_TRUE(dl > 256 * 1024 && dl < 256 * 1024 + 128); + ASSERT_NOT_NULL(strstr(page->docstring, "[page text cut at 262144 of ")); + bool tail = false; + for (int i = 0; i < r->doc_links.count; i++) { + tail = tail || strcmp(r->doc_links.items[i].raw, "src/tail/last.py") == 0; + } + ASSERT_TRUE(tail); + cbm_free_result(r); + free(pdf); + free(c.p); + PASS(); +} + TEST(pdf_snippet_is_page_text) { char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX"; ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); @@ -1222,6 +1264,7 @@ SUITE(doc_links_pdf) { RUN_TEST(pdf_scan_mentions); RUN_TEST(pdf_scan_join_linear); RUN_TEST(pdf_links_pipeline); + RUN_TEST(pdf_page_doc_bounded); RUN_TEST(pdf_snippet_is_page_text); RUN_TEST(pdf_links_incremental); }