diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index ec186c05a..855c460fe 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -75,6 +75,14 @@ typedef struct { uint64_t last_dirty_sig; /* committed dirty-state signature */ uint64_t pending_dirty_sig; /* observed at check time */ char pending_head[CBM_SZ_128]; /* HEAD observed at check time */ + /* Consecutive hard index failures (index_fn < 0). A hard error is + * usually persistent — a poisoned coordination endpoint, an unreadable + * DB — so retrying it at the plain poll interval re-forks a worker that + * fails identically, for as long as the daemon lives. #937 deliberately + * leaves the baseline uncommitted so the change is never lost; this + * decays the retry cadence so "never lost" does not also mean "retried + * forever". Reset to 0 by any successful reindex. */ + int index_failures; /* Hop from root_path up to the repository root ("" when they are the same), * from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so * the signature needs this to stat them. Resolved once at baseline. */ @@ -114,6 +122,16 @@ struct cbm_watcher { #define POLL_FILE_STEP 500 /* add 1s per this many files */ #define POLL_MAX_MS 60000 +/* Hard index-failure backoff. Doubling per consecutive failure, capped, so a + * persistently failing project costs ~12 attempts/hour instead of ~480 while + * still recovering on its own within the ceiling once the cause clears. The + * shift cap keeps the intermediate value well inside int64 for any interval. */ +#define INDEX_FAIL_SHIFT_MAX 6 +#define INDEX_FAIL_CEILING_MS 300000 /* 5 min */ +/* Log a distinct line once the failures are clearly not transient, so the + * daemon log names the stuck project instead of only repeating the warning. */ +#define INDEX_FAIL_SUSTAINED 10 + /* Stale-root pruning (#286): a watched project whose root directory stays * missing is pruned — its cached DB is deleted and the watch entry removed. * Deletion is destructive (the DB can hold user-authored data such as the @@ -143,6 +161,19 @@ static int64_t now_ns(void) { /* ── Adaptive interval ──────────────────────────────────────────── */ +int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) { + if (interval_ms < 0) { + interval_ms = 0; + } + if (consecutive_failures <= 0) { + return interval_ms; + } + int shift = + consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX; + int64_t delay = (int64_t)interval_ms << shift; + return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay; +} + int cbm_watcher_poll_interval_ms(int file_count) { int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC); if (ms > POLL_MAX_MS) { @@ -1429,6 +1460,7 @@ static void poll_project(const char *key, void *val, void *ud) { int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data); if (rc == 0) { ctx->reindexed++; + s->index_failures = 0; /* Commit the baselines OBSERVED AT CHECK TIME — the state whose * reindex just succeeded. A commit/edit landing during the * reindex is deliberately not absorbed: the next poll sees it @@ -1447,11 +1479,28 @@ static void poll_project(const char *key, void *val, void *ud) { /* Busy-skip: baseline stays uncommitted, next poll retries. */ cbm_log_info("watcher.index.retry", "project", s->project_name); } else { - cbm_log_warn("watcher.index.err", "project", s->project_name); + /* itoa_buf returns one shared per-thread buffer, so two of them + * in one call would print the same value twice. */ + char rc_text[CBM_SZ_32]; + char streak_text[CBM_SZ_32]; + if (s->index_failures < INT_MAX) { + s->index_failures++; + } + snprintf(rc_text, sizeof(rc_text), "%d", rc); + snprintf(streak_text, sizeof(streak_text), "%d", s->index_failures); + cbm_log_warn("watcher.index.err", "project", s->project_name, "rc", rc_text, + "consecutive", streak_text); + if (s->index_failures == INDEX_FAIL_SUSTAINED) { + cbm_log_warn("watcher.index.sustained_failure", "project", s->project_name, + "consecutive", streak_text); + } } } - s->next_poll_ns = ctx->now + ((int64_t)s->interval_ms * US_PER_MS); + /* Failures back off; success and busy-skip keep the adaptive cadence. */ + s->next_poll_ns = + ctx->now + + ((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS); } /* Callback to snapshot project state pointers into an array. */ diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 2a30cbb4f..18459afc5 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -94,6 +94,13 @@ int cbm_watcher_watch_count(cbm_watcher_t *w); /* Return the adaptive poll interval (ms) for a given file count. */ int cbm_watcher_poll_interval_ms(int file_count); +/* Return the delay (ms) before the next index attempt for a project with + * `consecutive_failures` consecutive hard index failures. Zero failures + * yields `interval_ms` unchanged; each further failure doubles the delay up + * to a fixed ceiling, so a permanently failing project stops re-forking a + * worker at the poll cadence without ever being abandoned. */ +int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures); + /* Classify a stat() errno observed on a watched project root: returns true * only for values that mean the root itself is gone (ENOENT, ENOTDIR) and * may count toward stale-root pruning (#286). Any other failure (EACCES, diff --git a/tests/test_watcher.c b/tests/test_watcher.c index 14b602be1..42dad66c9 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,57 @@ TEST(poll_interval_small) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * HARD INDEX-FAILURE BACKOFF + * ══════════════════════════════════════════════════════════════════ */ + +TEST(index_backoff_no_failures_keeps_interval) { + /* The success and busy-skip paths must be completely unaffected. */ + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 0), 5000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, 0), 60000); + PASS(); +} + +TEST(index_backoff_doubles_per_failure) { + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 1), 10000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 2), 20000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 3), 40000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 4), 80000); + PASS(); +} + +TEST(index_backoff_reaches_ceiling_and_stays) { + /* 5000 << 6 = 320000, above the 5-minute ceiling. The shift is capped + * too, so an arbitrarily long failure streak cannot overflow the + * intermediate value or wrap back to a short delay. */ + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 6), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 100), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, INT_MAX), 300000); + ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, INT_MAX), 300000); + PASS(); +} + +TEST(index_backoff_is_monotonic_and_bounded) { + /* The property that actually matters: the delay never decreases as the + * streak grows, and never exceeds the ceiling. This is what bounds the + * fork rate of a permanently failing project. */ + int previous = cbm_watcher_index_backoff_ms(5000, 0); + for (int failures = 1; failures < 200; failures++) { + int delay = cbm_watcher_index_backoff_ms(5000, failures); + ASSERT_TRUE(delay >= previous); + ASSERT_TRUE(delay <= 300000); + previous = delay; + } + PASS(); +} + +TEST(index_backoff_degenerate_inputs) { + ASSERT_EQ(cbm_watcher_index_backoff_ms(0, 5), 0); + ASSERT_EQ(cbm_watcher_index_backoff_ms(-1, 0), 0); + ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, -1), 5000); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * LIFECYCLE * ══════════════════════════════════════════════════════════════════ */ @@ -3143,6 +3195,11 @@ SUITE(watcher) { RUN_TEST(poll_interval_scaling); RUN_TEST(poll_interval_cap); RUN_TEST(poll_interval_small); + RUN_TEST(index_backoff_no_failures_keeps_interval); + RUN_TEST(index_backoff_doubles_per_failure); + RUN_TEST(index_backoff_reaches_ceiling_and_stays); + RUN_TEST(index_backoff_is_monotonic_and_bounded); + RUN_TEST(index_backoff_degenerate_inputs); /* Lifecycle */ RUN_TEST(watcher_create_free);