fix(rst): a section's text no longer reads an unwritten byte

rst_body cut a reST section's text at 500 bytes by setting a stop value
past the end, then backing off from out[RST_BODY_MAX] while that byte
looked like a UTF-8 continuation byte. That byte was never written, so
where the text ended (499 or 500 bytes) depended on what the arena memory
held, and when a multibyte character did not fit, bytes never written
could end up in the text.

Only whole characters are written now, and the text ends at the last one
written. The same django tree indexed twice gave doc_link_candidates rows
that differed in about 250 of 11,300 (through the section text's TF-IDF);
with this change two runs are identical, and so are the 6,594 sections.

The new test pins the cut at the boundary: a word that fills the 500 bytes,
a two-byte character that would cross them, a word cut at a character.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

internal/cbm/doclink_rst.c

@@ -1606,18 +1606,19 @@ static const char *rst_body(rst_doc_t *d, int from, int to) {
         return NULL;
     }
     int w = 0;
-    for (int k = from; k < to && w < RST_BODY_MAX; k++) {
+    bool full = false; /* the next character did not fit: the text ends before it */
+    for (int k = from; k < to && !full; k++) {
         const char *s = d->L[k].s;
         int len = d->L[k].len;
         int i = 0;
-        while (i < len && w < RST_BODY_MAX) {
+        while (i < len && !full) {
             int n;
             uint32_t cp = rst_cp(s, len, i, &n);
             if (rst_space_cp(cp)) {
                 i += n;
                 continue;
             }
-            if (w > 0) {
+            if (w > 0 && w < RST_BODY_MAX) {
                 out[w++] = ' ';
             }
             while (i < len) {
@@ -1626,7 +1627,7 @@ static const char *rst_body(rst_doc_t *d, int from, int to) {
                     break;
                 }
                 if (w + n > RST_BODY_MAX) {
-                    w = RST_BODY_MAX + 1; /* stop: the next character does not fit */
+                    full = true; /* only whole characters are written: none is cut */
                     break;
                 }
                 memcpy(out + w, s + i, (size_t)n);
@@ -1635,12 +1636,6 @@ static const char *rst_body(rst_doc_t *d, int from, int to) {
             }
         }
     }
-    if (w > RST_BODY_MAX) {
-        w = RST_BODY_MAX;
-        while (w > 0 && ((unsigned char)out[w] & 0xC0) == 0x80) {
-            w--;
-        }
-    }
     while (w > 0 && out[w - 1] == ' ') {
         w--;
     }

tests/test_doc_links_rst.c

@@ -558,9 +558,48 @@ TEST(rst_ship_gate) {
     PASS();
 }
 
+/* A section's text is cut before the first character that does not fit its
+ * 500 bytes: only whole characters are written, and the text ends at the last
+ * one written (it once tested an unwritten byte past the end, so where the
+ * text ended depended on what that memory held). */
+TEST(rst_section_text_cut) {
+    static const struct {
+        const char *tail;
+        const char *want_tail;
+    } cases[] = {
+        {" xxxxx tail", " xxxxx"},       /* a word that ends at byte 500 */
+        {" xxxx\xc3\xa9 more", " xxxx"}, /* a two-byte character that would cross it */
+        {" yyyyyyyy", " yyyyy"},         /* a word cut at a character */
+    };
+    for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {
+        char doc[1024];
+        char want[600];
+        int n = snprintf(doc, sizeof(doc), "Title\n=====\n\n");
+        int w = 0;
+        for (int k = 0; k < 99; k++) { /* 99 words: 494 bytes, nine per paragraph */
+            n += snprintf(doc + n, sizeof(doc) - (size_t)n, "%sabcd",
+                          k == 0       ? ""
+                          : k % 9 == 0 ? "\n\n"
+                                       : " ");
+            w += snprintf(want + w, sizeof(want) - (size_t)w, "%sabcd", k ? " " : "");
+        }
+        snprintf(doc + n, sizeof(doc) - (size_t)n, "%s\n", cases[c].tail);
+        snprintf(want + w, sizeof(want) - (size_t)w, "%s", cases[c].want_tail);
+        CBMFileResult *r = dm_extract(doc, CBM_LANG_RST, "docs/cut.rst");
+        ASSERT_NOT_NULL(r);
+        const CBMDefinition *t = rst_def(r, "Section", "Title");
+        ASSERT_NOT_NULL(t);
+        ASSERT_NOT_NULL(t->docstring);
+        ASSERT_STR_EQ(t->docstring, want);
+        cbm_free_result(r);
+    }
+    PASS();
+}
+
 SUITE(doc_links_rst) {
     dm_ship_held_families();
     RUN_TEST(rst_scan_structure);
+    RUN_TEST(rst_section_text_cut);
     RUN_TEST(rst_python_scope_blob);
     RUN_TEST(rst_links_pipeline);
     RUN_TEST(rst_c_member_owner);