fix(doclinks): removed declarations repair incrementally in TS, PHP, C, Perl

The TS/JS, PHP, C and Perl doc-link layers keep a scope blob per file but
had no scope_delta hook, so any change of a file's declarations made the
incremental closure repair decline (doc_scope_changed) and the index
rebuild in full: removing a TypeScript function forced a full run.

cbm_doclinks_scope_delta_lines compares two blobs line by line in order,
as C# does: a dropped record that declares one name is repaired file by
file (the name is reported, so the files whose references named it are
re-resolved); anything new or changed, and every record others resolve
through, stays a full run. The records that declare one name: TS D
(a top-level declaration), PHP F and K (a function, a member), C D of a
function, variable, enum constant, field or macro and A (a Doxygen
anchor), Perl S (a sub).

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/pipeline/doc_links.c

@@ -120,6 +120,64 @@ int cbm_doclinks_scope_delta(const char *stored, const char *fresh, cbm_doclink_
     return R->scope_delta(stored, fresh, removed, ud);
 }
 
+/* One line of a scope blob at *p, without its newline; false at the end. */
+static bool scope_next_line(const char **p, const char **line, size_t *len) {
+    if (!*p || !**p) {
+        return false;
+    }
+    const char *nl = strchr(*p, '\n');
+    *line = *p;
+    *len = nl ? (size_t)(nl - *p) : strlen(*p);
+    *p = nl ? nl + SKIP_ONE : *p + *len;
+    return true;
+}
+
+int cbm_doclinks_scope_delta_lines(const char *stored, const char *fresh,
+                                   cbm_doclink_removable_fn removable, cbm_doclink_name_fn removed,
+                                   void *ud) {
+    const char *sp = stored;
+    const char *fp = fresh;
+    const char *sl = NULL;
+    const char *fl = NULL;
+    size_t slen = 0;
+    size_t flen = 0;
+    bool hs = scope_next_line(&sp, &sl, &slen);
+    bool hf = scope_next_line(&fp, &fl, &flen);
+    while (hs || hf) {
+        if (hs && hf && slen == flen && memcmp(sl, fl, slen) == 0) {
+            hs = scope_next_line(&sp, &sl, &slen);
+            hf = scope_next_line(&fp, &fl, &flen);
+            continue;
+        }
+        const char *name = NULL;
+        size_t nlen = 0;
+        if (!hs || !removable(sl, slen, &name, &nlen) || nlen == 0) {
+            return CBM_DOCLINK_DELTA_GLOBAL; /* new, changed, or resolved through */
+        }
+        if (!removed || !removed(ud, name, nlen)) {
+            return CBM_NOT_FOUND;
+        }
+        hs = scope_next_line(&sp, &sl, &slen);
+    }
+    return CBM_DOCLINK_DELTA_LOCAL;
+}
+
+bool cbm_doclinks_scope_field(const char *line, size_t len, int k, const char **field,
+                              size_t *field_len) {
+    size_t s = 0;
+    for (int f = 0; f < k; f++) {
+        const char *tab = memchr(line + s, '\t', len - s);
+        if (!tab) {
+            return false;
+        }
+        s = (size_t)(tab - line) + SKIP_ONE;
+    }
+    const char *tab = memchr(line + s, '\t', len - s);
+    *field = line + s;
+    *field_len = tab ? (size_t)(tab - line) - s : len - s;
+    return true;
+}
+
 const char *cbm_doclinks_storable_scope(const char *scope, char **owned) {
     *owned = NULL;
     const cbm_doclink_resolver_t *R = resolver_of_scope(scope);

src/pipeline/doc_links.h

@@ -313,6 +313,25 @@ bool cbm_doclinks_is_scope_input(const char *rel_path);
 int cbm_doclinks_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed,
                              void *ud);
 
+/* A record of a scope blob that declares one name other files may reference
+ * and nothing others resolve THROUGH: true, with that name. */
+typedef bool (*cbm_doclink_removable_fn)(const char *line, size_t len, const char **name,
+                                         size_t *name_len);
+
+/* A scope_delta over the lines of two blobs, in order (a record may belong to
+ * the one above it): the same line in both is unchanged; a stored line the
+ * fresh blob dropped is LOCAL when `removable` names it -- the name goes to
+ * `removed` -- and GLOBAL otherwise; a line only the fresh blob has, new or
+ * changed, is GLOBAL. A cbm_doclink_delta_t, or CBM_NOT_FOUND when a name
+ * could not be recorded. */
+int cbm_doclinks_scope_delta_lines(const char *stored, const char *fresh,
+                                   cbm_doclink_removable_fn removable, cbm_doclink_name_fn removed,
+                                   void *ud);
+
+/* Field k (0: the record's kind) of a tab-separated scope line. */
+bool cbm_doclinks_scope_field(const char *line, size_t len, int k, const char **field,
+                              size_t *field_len);
+
 /* What is stored for a scope blob written in this run (the surface row's
  * `dl`, lsp_surface.c): the blob itself, or its language's rejected-scope
  * marker when that language's reader refuses it (scope_accepted), made into

src/pipeline/doc_links_c.c

@@ -471,6 +471,27 @@ static void cr_resolve_name(const cr_index_t *ix, int src, const CBMDocLink *lin
     }
 }
 
+/* A function, variable, enum constant, field or macro (D kind link name qn
+ * parent, kind f v e r m M) and a Doxygen anchor (A name) each declare one
+ * name. A type (t) is what members and Type::member are found through:
+ * GLOBAL, as is every new or changed record. */
+static bool cr_removable(const char *line, size_t len, const char **name, size_t *nlen) {
+    if (len <= PAIR_LEN || line[1] != '\t') {
+        return false;
+    }
+    if (line[0] == 'A') {
+        return cbm_doclinks_scope_field(line, len, 1, name, nlen);
+    }
+    return line[0] == 'D' && strchr("fverMm", line[PAIR_LEN]) && line[PAIR_LEN] &&
+           line[PAIR_LEN + SKIP_ONE] == '\t' &&
+           cbm_doclinks_scope_field(line, len, CBM_SZ_3, name, nlen);
+}
+
+static int cr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed,
+                          void *ud) {
+    return cbm_doclinks_scope_delta_lines(stored, fresh, cr_removable, removed, ud);
+}
+
 const cbm_doclink_resolver_t cbm_doclink_c_resolver = {
     .langs = {CBM_LANG_C, CBM_LANG_CPP},
     .lang_count = PAIR_LEN,
@@ -478,4 +499,5 @@ const cbm_doclink_resolver_t cbm_doclink_c_resolver = {
     .build = cr_build,
     .destroy = cr_destroy,
     .resolve = cr_resolve,
+    .scope_delta = cr_scope_delta,
 };

src/pipeline/doc_links_perl.c

@@ -26,9 +26,10 @@
  * Test code (t/, xt/, *.t, and cbm_is_test_file) is bound only from test code.
  * Every edge is exact: a module name and a heading are spelled out in full.
  *
- * The scope blob carries no lines (doclink_perl.c), and without a scope_delta
- * hook every change of a blob is a full re-resolution: a change that adds or
- * drops a package, a sub or a heading; an edit inside a sub changes no blob.
+ * The scope blob carries no lines (doclink_perl.c). A dropped sub is repaired
+ * file by file (pl_scope_delta); a change that adds a sub, or adds or drops a
+ * package or a heading, is a full re-resolution; an edit inside a sub changes
+ * no blob.
  */
 #include "pipeline/doc_links.h"
 #include "pipeline/pipeline_internal.h" /* cbm_pipeline_file_node */
@@ -479,6 +480,19 @@ static void pr_resolve(const void *index, void *state, int run_file, const CBMDo
     pr_edge(&c, f, pr_module_node(&c, f));
 }
 
+/* A sub (S pkg name) declares one name. A package (its index places every
+ * sub after it), a heading (a section is found by its text) and the
+ * parse-error mark change how others resolve: GLOBAL. */
+static bool pl_removable(const char *line, size_t len, const char **name, size_t *nlen) {
+    return len > PAIR_LEN && line[0] == 'S' && line[1] == '\t' &&
+           cbm_doclinks_scope_field(line, len, PAIR_LEN, name, nlen);
+}
+
+static int pl_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed,
+                          void *ud) {
+    return cbm_doclinks_scope_delta_lines(stored, fresh, pl_removable, removed, ud);
+}
+
 const cbm_doclink_resolver_t cbm_doclink_perl_resolver = {
     .langs = {CBM_LANG_PERL},
     .lang_count = 1,
@@ -486,4 +500,5 @@ const cbm_doclink_resolver_t cbm_doclink_perl_resolver = {
     .build = pr_build,
     .destroy = pr_destroy,
     .resolve = pr_resolve,
+    .scope_delta = pl_scope_delta,
 };

src/pipeline/doc_links_php.c

@@ -954,6 +954,35 @@ static void pr_resolve(const void *index, void *state, int run_file, const CBMDo
     pr_out(&c, res, out);
 }
 
+/* A top-level function (F fqn flags) or a member of the type above it (K kind
+ * name) declares one name; reported as a reference spells it, the last
+ * segment of a qualified name. A type, a supertype, a namespace block or a
+ * use import changes how others resolve: GLOBAL. */
+static bool pr_removable(const char *line, size_t len, const char **name, size_t *nlen) {
+    if (len <= PAIR_LEN || line[1] != '\t') {
+        return false;
+    }
+    if (line[0] == 'K') {
+        return cbm_doclinks_scope_field(line, len, PAIR_LEN, name, nlen);
+    }
+    if (line[0] != 'F' || !cbm_doclinks_scope_field(line, len, 1, name, nlen)) {
+        return false;
+    }
+    for (size_t i = *nlen; i > 0; i--) {
+        if ((*name)[i - SKIP_ONE] == '\\') {
+            *nlen -= i;
+            *name += i;
+            break;
+        }
+    }
+    return true;
+}
+
+static int pr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed,
+                          void *ud) {
+    return cbm_doclinks_scope_delta_lines(stored, fresh, pr_removable, removed, ud);
+}
+
 const cbm_doclink_resolver_t cbm_doclink_php_resolver = {
     .langs = {CBM_LANG_PHP},
     .lang_count = SKIP_ONE,
@@ -961,4 +990,5 @@ const cbm_doclink_resolver_t cbm_doclink_php_resolver = {
     .build = pr_build,
     .destroy = pr_destroy,
     .resolve = pr_resolve,
+    .scope_delta = pr_scope_delta,
 };

src/pipeline/doc_links_ts.c

@@ -1083,6 +1083,20 @@ static void tr_resolve(const void *index, void *state, int run_file, const CBMDo
     tr_out(&c, r);
 }
 
+/* A top-level declaration (D name flags) declares one name: other files reach
+ * it through an import of it, so a reference that did either bound it (an
+ * edge into this file) or names it in its unresolved row. Imports, exports
+ * and namespaces change how others resolve: GLOBAL. */
+static bool tr_removable(const char *line, size_t len, const char **name, size_t *nlen) {
+    return len > PAIR_LEN && line[0] == 'D' && line[1] == '\t' &&
+           cbm_doclinks_scope_field(line, len, 1, name, nlen);
+}
+
+static int tr_scope_delta(const char *stored, const char *fresh, cbm_doclink_name_fn removed,
+                          void *ud) {
+    return cbm_doclinks_scope_delta_lines(stored, fresh, tr_removable, removed, ud);
+}
+
 const cbm_doclink_resolver_t cbm_doclink_ts_resolver = {
     .langs = {CBM_LANG_TYPESCRIPT, CBM_LANG_TSX, CBM_LANG_JAVASCRIPT},
     .lang_count = CBM_SZ_3,
@@ -1090,4 +1104,5 @@ const cbm_doclink_resolver_t cbm_doclink_ts_resolver = {
     .build = tr_build,
     .destroy = tr_destroy,
     .resolve = tr_resolve,
+    .scope_delta = tr_scope_delta,
 };

tests/test_doc_mentions_c.c

@@ -287,7 +287,12 @@ TEST(doc_mentions_c_incremental_equals_full) {
     ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
     ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a"));
 
-    /* a becomes static: without a scope_delta hook that is a full run */
+    /* b goes: the files that name it are repaired, nothing else */
+    th_write_file(TH_PATH(repo, "src/lib.c"), "int a(void) { return 2; }\n");
+    ASSERT_EQ(dm_step(repo, inc_db, full_db, "b removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
+    ASSERT_TRUE(c_edge(inc_db, "src.use.user", "src.lib.a"));
+
+    /* a becomes static (and b is back): changed records, a full run */
     th_write_file(TH_PATH(repo, "src/lib.c"),
                   "static int a(void) { return 0; }\nint b(void) { return 1; }\n");
     ASSERT_EQ(dm_step(repo, inc_db, full_db, "made static", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0);

tests/test_doc_mentions_perl.c

@@ -229,8 +229,7 @@ TEST(doc_mentions_perl_incremental_equals_full) {
         dm_step(repo, inc_db, full_db, "module doc link", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
     ASSERT_TRUE(pl_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.pm.__file__"));
 
-    /* a new heading changes the file's scope blob: without a scope_delta hook
-     * that is a full run */
+    /* a new heading changes how a section is found: a full run */
     th_write_file(TH_PATH(repo, "lib/Knob.pm"),
                   "package Knob;\n\n=head1 NAME\n\nKnob, a knob - see L<Knob::Dial/spin> and\n"
                   "L<Knob::Dial>.\n\n=head1 SEE ALSO\n\nL<Knob::Dial/OVERVIEW>\n\n=cut\n\n"
@@ -249,6 +248,12 @@ TEST(doc_mentions_perl_incremental_equals_full) {
     int n = 0;
     dm_edge(inc_db, "lib.Knob.pm.__file__", "lib.Knob.Dial.spin", props, sizeof(props), &n);
     ASSERT_EQ(n, 0);
+
+    /* the sub goes: the files that name it are repaired, nothing else */
+    th_write_file(TH_PATH(repo, "lib/Knob/Dial.pm"),
+                  "package Knob::Dial;\n\n=head1 OVERVIEW\n\nThe dial.\n\n=cut\n\n1;\n");
+    ASSERT_EQ(dm_step(repo, inc_db, full_db, "sub removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR),
+              0);
     th_cleanup(tmp);
     PASS();
 }

tests/test_doc_mentions_php.c

@@ -425,10 +425,11 @@ TEST(doc_mentions_php_incremental_equals_full) {
     ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
     ASSERT_TRUE(php_edge(inc_db, "src.use.user", "src.lib.a"));
 
-    /* the function goes: without a scope_delta hook that is a full run */
+    /* the function goes: the files that name it are repaired, nothing else */
     th_write_file(TH_PATH(repo, "src/lib.php"), "<?php\nnamespace Lib;\nfunction b() {}\n");
-    ASSERT_EQ(dm_step(repo, inc_db, full_db, "function removed", CBM_INCREMENTAL_ROUTE_FORCED_FULL),
-              0);
+    ASSERT_EQ(
+        dm_step(repo, inc_db, full_db, "function removed", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR),
+        0);
     char props[512];
     int n = 0;
     dm_edge(inc_db, "src.use.user", "src.lib.a", props, sizeof(props), &n);

tests/test_doc_mentions_ts.c

@@ -307,7 +307,7 @@ TEST(doc_mentions_ts_incremental_equals_full) {
     ASSERT_EQ(dm_step(repo, inc_db, full_db, "body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
     ASSERT_TRUE(ts_edge(inc_db, "src.use.use", "src.lib.a"));
 
-    /* an export goes: without a scope_delta hook that is a full run */
+    /* an export goes: the declaration's record changes, a full run */
     th_write_file(TH_PATH(repo, "src/lib.ts"), "function a() {}\nexport function b() {}\n");
     ASSERT_EQ(dm_step(repo, inc_db, full_db, "export removed", CBM_INCREMENTAL_ROUTE_FORCED_FULL),
               0);