fix(watcher): back off re-forking a hard-failing index worker

A hard index failure (index_fn < 0) left the retry cadence untouched, so a
persistently failing project re-forked an index worker every poll for as
long as the daemon lived. Observed on 0.10.8/macOS: 2233 consecutive failed
workers over 4h43m, ~8/min, every one dying at the same worker-side
coordination seal. Read queries stayed healthy throughout, so nothing
surfaced to the user while no project could be indexed at all.

#937 deliberately leaves the baseline uncommitted on a failed reindex so the
change is retried rather than lost. That guarantee is kept; what changes is
the cadence. Each consecutive hard failure doubles the delay, capped at five
minutes, taking a permanently failing project from ~480 attempts/hour to
~12 while still recovering on its own once the cause clears. Busy-skip
(rc > 0) and success paths are untouched.

The delay arithmetic is a pure exported helper so it is unit-testable
without a clock. watcher.index.err now carries rc and the consecutive-
failure count, and a distinct watcher.index.sustained_failure names a
project whose failures are clearly not transient.

Note: itoa_buf returns a single shared per-thread buffer, so the two-value
log line uses local buffers rather than two itoa_buf calls.

Refs #2015

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf
Signed-off-by: Shane McCarron <shane.mccarron@corvexconnect.com>

src/watcher/watcher.c

@@ -75,6 +75,14 @@ typedef struct {
     uint64_t last_dirty_sig;       /* committed dirty-state signature */
     uint64_t pending_dirty_sig;    /* observed at check time */
     char pending_head[CBM_SZ_128]; /* HEAD observed at check time */
+    /* Consecutive hard index failures (index_fn < 0). A hard error is
+     * usually persistent — a poisoned coordination endpoint, an unreadable
+     * DB — so retrying it at the plain poll interval re-forks a worker that
+     * fails identically, for as long as the daemon lives. #937 deliberately
+     * leaves the baseline uncommitted so the change is never lost; this
+     * decays the retry cadence so "never lost" does not also mean "retried
+     * forever". Reset to 0 by any successful reindex. */
+    int index_failures;
     /* Hop from root_path up to the repository root ("" when they are the same),
      * from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so
      * the signature needs this to stat them. Resolved once at baseline. */
@@ -114,6 +122,16 @@ struct cbm_watcher {
 #define POLL_FILE_STEP 500 /* add 1s per this many files */
 #define POLL_MAX_MS 60000
 
+/* Hard index-failure backoff. Doubling per consecutive failure, capped, so a
+ * persistently failing project costs ~12 attempts/hour instead of ~480 while
+ * still recovering on its own within the ceiling once the cause clears. The
+ * shift cap keeps the intermediate value well inside int64 for any interval. */
+#define INDEX_FAIL_SHIFT_MAX 6
+#define INDEX_FAIL_CEILING_MS 300000 /* 5 min */
+/* Log a distinct line once the failures are clearly not transient, so the
+ * daemon log names the stuck project instead of only repeating the warning. */
+#define INDEX_FAIL_SUSTAINED 10
+
 /* Stale-root pruning (#286): a watched project whose root directory stays
  * missing is pruned — its cached DB is deleted and the watch entry removed.
  * Deletion is destructive (the DB can hold user-authored data such as the
@@ -143,6 +161,19 @@ static int64_t now_ns(void) {
 
 /* ── Adaptive interval ──────────────────────────────────────────── */
 
+int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) {
+    if (interval_ms < 0) {
+        interval_ms = 0;
+    }
+    if (consecutive_failures <= 0) {
+        return interval_ms;
+    }
+    int shift =
+        consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX;
+    int64_t delay = (int64_t)interval_ms << shift;
+    return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay;
+}
+
 int cbm_watcher_poll_interval_ms(int file_count) {
     int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC);
     if (ms > POLL_MAX_MS) {
@@ -1429,6 +1460,7 @@ static void poll_project(const char *key, void *val, void *ud) {
         int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data);
         if (rc == 0) {
             ctx->reindexed++;
+            s->index_failures = 0;
             /* Commit the baselines OBSERVED AT CHECK TIME — the state whose
              * reindex just succeeded. A commit/edit landing during the
              * reindex is deliberately not absorbed: the next poll sees it
@@ -1447,11 +1479,28 @@ static void poll_project(const char *key, void *val, void *ud) {
             /* Busy-skip: baseline stays uncommitted, next poll retries. */
             cbm_log_info("watcher.index.retry", "project", s->project_name);
         } else {
-            cbm_log_warn("watcher.index.err", "project", s->project_name);
+            /* itoa_buf returns one shared per-thread buffer, so two of them
+             * in one call would print the same value twice. */
+            char rc_text[CBM_SZ_32];
+            char streak_text[CBM_SZ_32];
+            if (s->index_failures < INT_MAX) {
+                s->index_failures++;
+            }
+            snprintf(rc_text, sizeof(rc_text), "%d", rc);
+            snprintf(streak_text, sizeof(streak_text), "%d", s->index_failures);
+            cbm_log_warn("watcher.index.err", "project", s->project_name, "rc", rc_text,
+                         "consecutive", streak_text);
+            if (s->index_failures == INDEX_FAIL_SUSTAINED) {
+                cbm_log_warn("watcher.index.sustained_failure", "project", s->project_name,
+                             "consecutive", streak_text);
+            }
         }
     }
 
-    s->next_poll_ns = ctx->now + ((int64_t)s->interval_ms * US_PER_MS);
+    /* Failures back off; success and busy-skip keep the adaptive cadence. */
+    s->next_poll_ns =
+        ctx->now +
+        ((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS);
 }
 
 /* Callback to snapshot project state pointers into an array. */

src/watcher/watcher.h

@@ -94,6 +94,13 @@ int cbm_watcher_watch_count(cbm_watcher_t *w);
 /* Return the adaptive poll interval (ms) for a given file count. */
 int cbm_watcher_poll_interval_ms(int file_count);
 
+/* Return the delay (ms) before the next index attempt for a project with
+ * `consecutive_failures` consecutive hard index failures. Zero failures
+ * yields `interval_ms` unchanged; each further failure doubles the delay up
+ * to a fixed ceiling, so a permanently failing project stops re-forking a
+ * worker at the poll cadence without ever being abandoned. */
+int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures);
+
 /* Classify a stat() errno observed on a watched project root: returns true
  * only for values that mean the root itself is gone (ENOENT, ENOTDIR) and
  * may count toward stale-root pruning (#286). Any other failure (EACCES,

tests/test_watcher.c

@@ -14,6 +14,7 @@
 #include <watcher/watcher.h>
 #include <store/store.h>
 #include <errno.h>
+#include <limits.h>
 #include <stdatomic.h>
 #include <signal.h>
 #include <string.h>
@@ -85,6 +86,57 @@ TEST(poll_interval_small) {
     PASS();
 }
 
+/* ══════════════════════════════════════════════════════════════════
+ *  HARD INDEX-FAILURE BACKOFF
+ * ══════════════════════════════════════════════════════════════════ */
+
+TEST(index_backoff_no_failures_keeps_interval) {
+    /* The success and busy-skip paths must be completely unaffected. */
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 0), 5000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, 0), 60000);
+    PASS();
+}
+
+TEST(index_backoff_doubles_per_failure) {
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 1), 10000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 2), 20000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 3), 40000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 4), 80000);
+    PASS();
+}
+
+TEST(index_backoff_reaches_ceiling_and_stays) {
+    /* 5000 << 6 = 320000, above the 5-minute ceiling. The shift is capped
+     * too, so an arbitrarily long failure streak cannot overflow the
+     * intermediate value or wrap back to a short delay. */
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 6), 300000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 100), 300000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, INT_MAX), 300000);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, INT_MAX), 300000);
+    PASS();
+}
+
+TEST(index_backoff_is_monotonic_and_bounded) {
+    /* The property that actually matters: the delay never decreases as the
+     * streak grows, and never exceeds the ceiling. This is what bounds the
+     * fork rate of a permanently failing project. */
+    int previous = cbm_watcher_index_backoff_ms(5000, 0);
+    for (int failures = 1; failures < 200; failures++) {
+        int delay = cbm_watcher_index_backoff_ms(5000, failures);
+        ASSERT_TRUE(delay >= previous);
+        ASSERT_TRUE(delay <= 300000);
+        previous = delay;
+    }
+    PASS();
+}
+
+TEST(index_backoff_degenerate_inputs) {
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(0, 5), 0);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(-1, 0), 0);
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, -1), 5000);
+    PASS();
+}
+
 /* ══════════════════════════════════════════════════════════════════
  *  LIFECYCLE
  * ══════════════════════════════════════════════════════════════════ */
@@ -3143,6 +3195,11 @@ SUITE(watcher) {
     RUN_TEST(poll_interval_scaling);
     RUN_TEST(poll_interval_cap);
     RUN_TEST(poll_interval_small);
+    RUN_TEST(index_backoff_no_failures_keeps_interval);
+    RUN_TEST(index_backoff_doubles_per_failure);
+    RUN_TEST(index_backoff_reaches_ceiling_and_stays);
+    RUN_TEST(index_backoff_is_monotonic_and_bounded);
+    RUN_TEST(index_backoff_degenerate_inputs);
 
     /* Lifecycle */
     RUN_TEST(watcher_create_free);