pdf: a chain of objects costs no stack per object

Loading a stream whose /Length referred to another stream loaded that
stream first, and its /Length the next one: one nested call per object.
Loading an object of an object stream whose /Length (or filter) lay in
another object stream opened that one inside the first, again one call
per stream. A crafted chain overflowed the stack (D1 of the document
layer security review).

A /Length is now read as a value only: an object that is a stream there
gives no length (the stream is then read up to its endstream, as for any
wrong length) and is marked, so a second reference costs nothing. The
objects that open an object stream are no objects of an object stream
(ISO 32000-1, 7.5.7); one that is, is refused and counted
(objstm_nested). Both rules leave the nesting constant; nothing is cut
by a limit.

Test: pdf_extract_object_chains extracts a 20,000-deep /Length chain and
a 3,000-deep object-stream chain on a 256 KiB thread stack (stack
overflow before; each half fails alone when its rule is reverted). The
749-PDF dev corpus extracts byte-identically to the prototype as before
(720 of 720 documents, 1,492 of 1,492 pages).

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

internal/cbm/pdf/pdf.h

@@ -38,6 +38,7 @@ typedef struct {
     int xref_reconstructed;      /* cross-reference rebuilt from a scan of the file */
     int xref_offset_fixed;       /* an xref section found near a wrong offset */
     int stream_length_recovered; /* a stream's /Length was wrong */
+    int objstm_nested;           /* opening an object stream needed an object of one */
     int flate_recovered;         /* a damaged Flate stream decoded in part */
     int stream_guard_hits;       /* a decoded stream cut at 512 MiB */
     int filters_unsupported;     /* a stream with a filter this reader does not decode */

internal/cbm/pdf/pdf_internal.h

@@ -208,6 +208,7 @@ typedef struct {
     pdf_val_t *obj;
     bool cached;
     bool loading;
+    bool not_length;   /* read as a stream's /Length, it was a stream: no length */
     pdf_objstm_t *stm; /* this object as a decoded object stream */
 } pdf_xent_t;
 
@@ -248,6 +249,7 @@ struct pdf_doc {
     pdf_xref_t xref;
     pdf_val_t *trailer; /* a dict */
     bool reconstructed;
+    int objstm_opening; /* object streams being opened, one inside the other */
     /* every "N G obj" in file order, built on first need */
     pdf_hdr_t *hdrs;
     int nhdrs;

internal/cbm/pdf/pdf_obj.c

@@ -1072,10 +1072,13 @@ static bool endstream_follows(const pdf_doc_t *d, size_t q) {
     return false;
 }
 
-/* The object at off (or off + hdr). NULL: no matching header there. */
-static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,
-                                    bool *ok) {
+/* The value of the object at off (or off + hdr), without its stream; *stream_at
+ * is where a "stream" keyword after the value starts, 0 when none follows. NULL:
+ * no matching header there. */
+static pdf_val_t *indirect_value_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,
+                                    bool *ok, size_t *stream_at) {
     *ok = false;
+    *stream_at = 0;
     size_t at = 0;
     bool found = false;
     int64_t num = 0;
@@ -1106,20 +1109,75 @@ static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect,
     size_t p2 = p;
     pdf_tok_t t2;
     lex_peek(d, s, len, &p2, &t2);
-    if (!(pdf_kw_is(&t2, "stream") && val->kind == PV_DICT)) {
+    if (pdf_kw_is(&t2, "stream") && val->kind == PV_DICT) {
+        *stream_at = p2;
+    }
+    return val;
+}
+
+static int64_t hdr_first_exact(pdf_doc_t *d, int64_t num, int64_t gen);
+
+/* An object a stream's /Length names, read as a value only and cached as
+ * pdf_get caches it. An object that is a stream there gives no length and is
+ * marked: loading it would read ITS /Length first, one nested call per stream
+ * of a chain of such streams. */
+static pdf_val_t *length_object(pdf_doc_t *d, int64_t num) {
+    pdf_xent_t *e = xent(d, num);
+    bool ok;
+    size_t stream_at;
+    pdf_val_t *v = indirect_value_at(d, e->a, true, num, &ok, &stream_at);
+    if (!ok && !d->reconstructed) {
+        int64_t off = hdr_first_exact(d, num, e->b);
+        if (off >= 0) {
+            v = indirect_value_at(d, off, true, num, &ok, &stream_at);
+        }
+    }
+    e = xent(d, num);
+    if (ok && stream_at) {
+        e->not_length = true;
+        return NULL;
+    }
+    e->cached = true;
+    e->obj = ok ? v : NULL;
+    return e->obj;
+}
+
+/* A stream's /Length through its references (at most PDF_MAX_RESOLVE, as
+ * pdf_resolve). */
+static pdf_val_t *length_value(pdf_doc_t *d, pdf_val_t *v) {
+    for (int depth = 0; v && v->kind == PV_REF; depth++) {
+        pdf_xent_t *e = xent(d, v->ref_num);
+        if (depth >= PDF_MAX_RESOLVE || !e || e->loading || e->not_length) {
+            return NULL;
+        }
+        if (e->cached) {
+            v = e->obj;
+        } else if (e->type == 1) {
+            v = length_object(d, v->ref_num);
+        } else {
+            v = pdf_get(d, v->ref_num);
+        }
+    }
+    return v;
+}
+
+/* The object at off (or off + hdr), a stream with its raw bytes when one
+ * follows. NULL: no matching header there. */
+static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,
+                                    bool *ok) {
+    size_t st;
+    pdf_val_t *val = indirect_value_at(d, off, has_expect, expect, ok, &st);
+    if (!val || !st) {
         return val;
     }
-    size_t st = p2;
+    const unsigned char *s = d->data;
+    size_t len = d->len;
     if (st + 1 < len && s[st] == '\r' && s[st + 1] == '\n') {
         st += 2;
     } else if (st < len && (s[st] == '\n' || s[st] == '\r')) {
         st += 1;
     }
-    pdf_val_t *ln = pdf_dget_raw(val, "Length");
-    if (ln && ln->kind == PV_REF) {
-        pdf_xent_t *e = xent(d, ln->ref_num);
-        ln = (e && e->loading) ? NULL : pdf_resolve(d, ln);
-    }
+    pdf_val_t *ln = length_value(d, pdf_dget_raw(val, "Length"));
     const unsigned char *raw = NULL;
     size_t raw_len = 0;
     if (ln && ln->kind == PV_NUM && ln->is_int && ln->inum >= 0 &&
@@ -1318,6 +1376,14 @@ pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num) {
     if (e->loading) {
         return NULL;
     }
+    if (e->type == 2 && d->objstm_opening > 0) {
+        /* The objects that open an object stream (its /Length, its filters) are
+         * no objects of an object stream (ISO 32000-1, 7.5.7): one that is would
+         * open the next stream inside this one, one nested call per stream of a
+         * chain. */
+        d->counts->objstm_nested++;
+        return NULL;
+    }
     e->loading = true;
     uint8_t type = e->type;
     int64_t a = e->a;
@@ -1366,7 +1432,7 @@ pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v) {
     return v;
 }
 
-static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {
+static pdf_objstm_t *objstm_open(pdf_doc_t *d, int64_t stmnum) {
     pdf_xent_t *e = xent(d, stmnum);
     if (e && e->stm) {
         return e->stm;
@@ -1421,6 +1487,13 @@ static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {
     return sm;
 }
 
+static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {
+    d->objstm_opening++;
+    pdf_objstm_t *sm = objstm_open(d, stmnum);
+    d->objstm_opening--;
+    return sm;
+}
+
 /* ── Cross-reference ─────────────────────────────────────────────── */
 
 static void xref_free(pdf_xref_t *x) {

tests/test_doc_links_pdf.c

@@ -16,6 +16,7 @@
 
 #include "cbm.h"
 #include "doclink.h"
+#include "foundation/compat_thread.h"
 #include "mcp/mcp.h"
 #include "pdf/pdf.h"
 #include "pipeline/pipeline.h"
@@ -134,6 +135,177 @@ static bool tp_extract(const unsigned char *pdf, size_t len, tp_text_t *t) {
     return cbm_pdf_extract(pdf, len, &t->arena, &t->r);
 }
 
+/* The smallest stack a thread of this program runs on (the daemon's). */
+#define TP_SMALL_STACK (256 * 1024)
+
+typedef struct {
+    const unsigned char *pdf;
+    size_t len;
+    tp_text_t *t;
+    bool ok;
+} tp_job_t;
+
+static void *tp_extract_job(void *arg) {
+    tp_job_t *j = (tp_job_t *)arg;
+    j->ok = tp_extract(j->pdf, j->len, j->t);
+    return NULL;
+}
+
+/* tp_extract on a thread with the smallest stack: an input that nests calls
+ * per object crashes here long before it would on a worker's 8 MiB. */
+static bool tp_extract_small_stack(const unsigned char *pdf, size_t len, tp_text_t *t) {
+    tp_job_t j = {pdf, len, t, false};
+    cbm_thread_t th;
+    if (cbm_thread_create(&th, TP_SMALL_STACK, tp_extract_job, &j) != 0) {
+        return false;
+    }
+    cbm_thread_join(&th);
+    return j.ok;
+}
+
+/* "N 0 obj\n" for object n at the end of b; records its offset. */
+static void tp_obj_head(tp_buf_t *b, size_t *off, int n) {
+    char line[64];
+    off[n] = b->n;
+    snprintf(line, sizeof(line), "%d 0 obj\n", n);
+    tp_puts(b, line);
+}
+
+/* A classic xref table for objects 1..n at off[] and the trailer. */
+static void tp_classic_tail(tp_buf_t *b, const size_t *off, int n) {
+    char line[128];
+    size_t xref = b->n;
+    snprintf(line, sizeof(line), "xref\n0 %d\n0000000000 65535 f \n", n + 1);
+    tp_puts(b, line);
+    for (int i = 1; i <= n; i++) {
+        snprintf(line, sizeof(line), "%010zu 00000 n \n", off[i]);
+        tp_puts(b, line);
+    }
+    snprintf(line, sizeof(line), "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%zu\n%%%%EOF\n",
+             n + 1, xref);
+    tp_puts(b, line);
+}
+
+#define TP_PAGE_F1_5 \
+    "<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"
+
+/* A page whose content stream's /Length names a stream whose /Length names
+ * the next one, `depth` streams deep (the last one's /Length is a number). */
+static unsigned char *tp_length_chain(int depth, size_t *out_len) {
+    int n = 4 + depth;
+    size_t *off = (size_t *)calloc((size_t)n + 1, sizeof(size_t));
+    tp_buf_t b = {0};
+    tp_puts(&b, "%PDF-1.7\n");
+    const char *head[4] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5,
+                           TP_HELV};
+    for (int i = 1; i <= 4; i++) {
+        tp_obj_head(&b, off, i);
+        tp_puts(&b, head[i - 1]);
+        tp_puts(&b, "\nendobj\n");
+    }
+    char line[96];
+    for (int i = 5; i <= n; i++) {
+        tp_obj_head(&b, off, i);
+        if (i < n) {
+            snprintf(line, sizeof(line), "<< /Length %d 0 R >>\nstream\n", i + 1);
+        } else {
+            snprintf(line, sizeof(line), "<< /Length 1 >>\nstream\n");
+        }
+        tp_puts(&b, line);
+        tp_puts(&b, i == 5 ? "BT /F1 12 Tf 72 700 Td (chain) Tj ET" : "x");
+        tp_puts(&b, "\nendstream\nendobj\n");
+    }
+    tp_classic_tail(&b, off, n);
+    free(off);
+    *out_len = b.n;
+    return b.p;
+}
+
+/* The page object lies in object stream C1, whose /Length lies in object
+ * stream C2, whose /Length lies in C3, ... `depth` streams deep; a
+ * cross-reference stream places them. Objects: 1-5 as tp_simple (3 in C1),
+ * C_i = 5 + i, L_i (C_i's length, in C_{i+1}) = 5 + depth + i, the xref stream
+ * last. */
+static unsigned char *tp_objstm_chain(int depth, size_t *out_len) {
+    int x = 5 + 2 * depth; /* the xref stream */
+    size_t *off = (size_t *)calloc((size_t)x + 1, sizeof(size_t));
+    size_t *data_len = (size_t *)calloc((size_t)depth + 1, sizeof(size_t));
+    tp_buf_t b = {0};
+    tp_puts(&b, "%PDF-1.7\n");
+    const char *plain[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, TP_HELV,
+                            NULL};
+    char line[160];
+    for (int i = 1; i <= 5; i++) {
+        if (i == 3) {
+            continue;
+        }
+        tp_obj_head(&b, off, i);
+        if (i == 5) {
+            const char *content = "BT /F1 12 Tf 72 700 Td (objstm chain) Tj ET";
+            snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content));
+            tp_puts(&b, line);
+            tp_puts(&b, content);
+            tp_puts(&b, "\nendstream");
+        } else {
+            tp_puts(&b, plain[i - 1]);
+        }
+        tp_puts(&b, "\nendobj\n");
+    }
+    for (int i = 1; i <= depth; i++) {
+        char hdr[48];
+        char body[160];
+        if (i == 1) {
+            snprintf(hdr, sizeof(hdr), "3 0 ");
+            snprintf(body, sizeof(body), "%s", TP_PAGE_F1_5);
+        } else {
+            snprintf(hdr, sizeof(hdr), "%d 0 ", 5 + depth + i - 1);
+            snprintf(body, sizeof(body), "%zu", data_len[i - 1]);
+        }
+        data_len[i] = strlen(hdr) + strlen(body);
+        tp_obj_head(&b, off, 5 + i);
+        if (i < depth) {
+            snprintf(line, sizeof(line),
+                     "<< /Type /ObjStm /N 1 /First %zu /Length %d 0 R >>\nstream\n", strlen(hdr),
+                     5 + depth + i);
+        } else {
+            snprintf(line, sizeof(line),
+                     "<< /Type /ObjStm /N 1 /First %zu /Length %zu >>\nstream\n", strlen(hdr),
+                     data_len[i]);
+        }
+        tp_puts(&b, line);
+        tp_puts(&b, hdr);
+        tp_puts(&b, body);
+        tp_puts(&b, "\nendstream\nendobj\n");
+    }
+    off[x] = b.n;
+    /* rows W [1 4 2]: type, offset or containing stream, generation or index */
+    size_t nrows = (size_t)x + 1;
+    unsigned char *rows = (unsigned char *)calloc(nrows, 7);
+    for (int i = 1; i <= x; i++) {
+        unsigned char *r = rows + (size_t)i * 7;
+        bool in_stm = i == 3 || (i > 5 + depth && i < x);
+        uint32_t f2 = in_stm ? (uint32_t)(i == 3 ? 6 : i - depth + 1) : (uint32_t)off[i];
+        r[0] = in_stm ? 2 : 1;
+        r[1] = (unsigned char)(f2 >> 24);
+        r[2] = (unsigned char)(f2 >> 16);
+        r[3] = (unsigned char)(f2 >> 8);
+        r[4] = (unsigned char)f2;
+    }
+    snprintf(line, sizeof(line),
+             "%d 0 obj\n<< /Type /XRef /Size %d /W [1 4 2] /Root 1 0 R /Length %zu >>\nstream\n", x,
+             x + 1, nrows * 7);
+    tp_puts(&b, line);
+    tp_put(&b, rows, nrows * 7);
+    tp_puts(&b, "\nendstream\nendobj\n");
+    snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", off[x]);
+    tp_puts(&b, line);
+    free(rows);
+    free(data_len);
+    free(off);
+    *out_len = b.n;
+    return b.p;
+}
+
 /* ── extractor ───────────────────────────────────────────────────── */
 
 TEST(pdf_extract_layout) {
@@ -478,6 +650,32 @@ TEST(pdf_extract_filters_and_hostile) {
     PASS();
 }
 
+/* A chain of objects, each needed to load the one before, costs no stack per
+ * object: a stream's /Length is read as a value (a stream there is no length),
+ * and the objects that open an object stream are no objects of one. Run on the
+ * smallest stack in the program. */
+TEST(pdf_extract_object_chains) {
+    size_t len;
+    tp_text_t t;
+    unsigned char *pdf = tp_length_chain(20000, &len);
+    ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t));
+    ASSERT_EQ(t.r.status, CBM_PDF_OK);
+    ASSERT_EQ(t.r.npages, 1);
+    ASSERT_STR_EQ(t.r.pages[0].text, "chain");
+    ASSERT_TRUE(t.r.counts.stream_length_recovered >= 1);
+    cbm_arena_destroy(&t.arena);
+    free(pdf);
+    pdf = tp_objstm_chain(3000, &len);
+    ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t));
+    ASSERT_EQ(t.r.status, CBM_PDF_OK);
+    ASSERT_EQ(t.r.npages, 1);
+    ASSERT_STR_EQ(t.r.pages[0].text, "objstm chain");
+    ASSERT_TRUE(t.r.counts.objstm_nested >= 1);
+    cbm_arena_destroy(&t.arena);
+    free(pdf);
+    PASS();
+}
+
 /* ── scanner (expected: the field-test scanner on the same text) ─── */
 
 TEST(pdf_scan_mentions) {
@@ -786,6 +984,7 @@ SUITE(doc_links_pdf) {
     RUN_TEST(pdf_extract_fonts);
     RUN_TEST(pdf_extract_structure);
     RUN_TEST(pdf_extract_filters_and_hostile);
+    RUN_TEST(pdf_extract_object_chains);
     RUN_TEST(pdf_scan_mentions);
     RUN_TEST(pdf_links_pipeline);
     RUN_TEST(pdf_snippet_is_page_text);