pdf: a page's docstring keeps 256 KiB of its text and says so

A PDF page's Section carried the whole page text as its docstring, which
is also what get_code_snippet returns for it: a crafted text layer
stored megabytes per node and handed them to an agent whole (D6 of the
document layer security review). Markdown, reStructuredText and AsciiDoc
sections keep a short docstring already.

A page's docstring is now its text up to 256 KiB, cut on a character
boundary and followed by a line that says where it was cut and how long
the text is. Pages hold a few kilobytes; the mentions are still read
from the whole text.

Test: pdf_page_doc_bounded: a 315 KB page keeps 256 KiB plus the note,
and a path written at its end is still a mention (the whole text is
stored with the bound reverted).

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

internal/cbm/doc_pdf.c

@@ -45,6 +45,7 @@ enum {
     PDF_PAGE_NAME = 32,
     PDF_LIST_MIN = 64,
     PDF_U8_MAX = 4,
+    PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */
 };
 
 #define PDF_JOIN_SEP '\x1f'  /* between a join's forms */
@@ -1321,6 +1322,23 @@ static bool fixture_pdf(const char *rel_path) {
     }
 }
 
+/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes
+ * cut on a character boundary and then said so. A page holds a few kilobytes;
+ * only a crafted text layer reaches the bound, and the mentions are still read
+ * from the whole text. */
+static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) {
+    if (pg->len <= PDF_PAGE_DOC_MAX) {
+        return pg->text;
+    }
+    size_t cut = PDF_PAGE_DOC_MAX;
+    while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) {
+        cut--; /* not inside a UTF-8 sequence */
+    }
+    const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut,
+                                        pg->text, cut, pg->len);
+    return doc ? doc : pg->text;
+}
+
 void cbm_pdf_extract_document(CBMExtractCtx *ctx) {
     CBMFileResult *result = ctx->result;
     CBMArena *a = ctx->arena;
@@ -1364,7 +1382,7 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx) {
         def.start_line = page;
         def.end_line = page;
         def.is_exported = true;
-        def.docstring = pr.pages[i].len ? pr.pages[i].text : NULL;
+        def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL;
         def.extra_props = kv;
         cbm_defs_push(&result->defs, a, def);
         if (!fixture && pr.pages[i].len) {

tests/test_doc_links_pdf.c

@@ -1133,6 +1133,48 @@ TEST(pdf_links_pipeline) {
     PASS();
 }
 
+/* A page's docstring keeps 256 KiB of its text and says where it was cut; a
+ * page's mentions still come from the whole text. */
+TEST(pdf_page_doc_bounded) {
+    enum { WORDS = 45000 }; /* 315 KB of text */
+    tp_buf_t c = {0};
+    tp_puts(&c, "BT /F1 12 Tf 72 700 Td (");
+    for (int i = 0; i < WORDS; i++) {
+        tp_puts(&c, "filler ");
+    }
+    tp_puts(&c, "src/tail/last.py) Tj ET");
+    tp_obj_t o[5] = {
+        {TP_CATALOG, NULL, 0},   {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0},
+        {TP_PAGE_F1_5, NULL, 0}, {TP_HELV, NULL, 0},
+        {"", c.p, c.n},
+    };
+    size_t len;
+    unsigned char *pdf = tp_pdf(o, 5, NULL, &len);
+    CBMFileResult *r =
+        cbm_extract_file((const char *)pdf, (int)len, CBM_LANG_PDF, "p", "doc.pdf", 0, NULL, NULL);
+    ASSERT_NOT_NULL(r);
+    const CBMDefinition *page = NULL;
+    for (int i = 0; i < r->defs.count; i++) {
+        if (strcmp(r->defs.items[i].label, "Section") == 0) {
+            page = &r->defs.items[i];
+        }
+    }
+    ASSERT_NOT_NULL(page);
+    ASSERT_NOT_NULL(page->docstring);
+    size_t dl = strlen(page->docstring);
+    ASSERT_TRUE(dl > 256 * 1024 && dl < 256 * 1024 + 128);
+    ASSERT_NOT_NULL(strstr(page->docstring, "[page text cut at 262144 of "));
+    bool tail = false;
+    for (int i = 0; i < r->doc_links.count; i++) {
+        tail = tail || strcmp(r->doc_links.items[i].raw, "src/tail/last.py") == 0;
+    }
+    ASSERT_TRUE(tail);
+    cbm_free_result(r);
+    free(pdf);
+    free(c.p);
+    PASS();
+}
+
 TEST(pdf_snippet_is_page_text) {
     char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX";
     ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
@@ -1222,6 +1264,7 @@ SUITE(doc_links_pdf) {
     RUN_TEST(pdf_scan_mentions);
     RUN_TEST(pdf_scan_join_linear);
     RUN_TEST(pdf_links_pipeline);
+    RUN_TEST(pdf_page_doc_bounded);
     RUN_TEST(pdf_snippet_is_page_text);
     RUN_TEST(pdf_links_incremental);
 }