Merge pull request #1666 from DeusData/fix/linux-arena-eager-commit

fix(mem): restore mimalloc's Linux arena-commit default (#1654)

src/foundation/mem.c

@@ -291,7 +291,36 @@ void cbm_mem_init_with_cap(double ram_fraction, size_t hard_cap_bytes) {
     }
 #endif
 
+    /* Lazy arena commit costs address-space FRAGMENTATION, and on Linux we now
+     * pay it for every allocation in the process. mimalloc commits a range with
+     * mprotect(PROT_READ|PROT_WRITE) over a sub-range of a PROT_NONE reservation
+     * (prim/unix/prim.c), and each partial commit SPLITS the reserved VMA. While
+     * mimalloc only served the bound populations (sqlite, tree_sitter) that was a
+     * handful of mappings. Since #1360 routed ordinary malloc/new through
+     * mimalloc on Linux, an index worker peaked at ~22k mappings against
+     * v0.9.0's 10 (Go corpus, 18 workers). The count tracks CONCURRENCY, not
+     * corpus size — sampled mid-run it was 999 at 1 worker, 8460 at 4 and 11965
+     * at 18, while v0.9.0 stayed at 10 regardless — so every worker thread
+     * fragments the address space independently.
+     *
+     * Two consequences, both reported as #1654 on a 96-CPU/376 GB host: the
+     * mmap/mprotect churn serialises on the kernel's per-process mmap_lock (the
+     * reporter saw 1.2% of extraction in 45 minutes, where v0.9.0 finished the
+     * tree in ~13), and the VMA count climbs toward vm.max_map_count, after
+     * which mmap fails for ANY size — hence mimalloc reporting it "cannot
+     * allocate" 10 KB while `free -g` still showed 246 GB available.
+     *
+     * mimalloc's own default is 2, meaning "eager-commit arenas only on an OS
+     * with overcommit (i.e. linux)" — precisely because commit is free there
+     * until the pages are touched. Overriding it to 0 opted Linux out of the
+     * default written for it. Restore the default on Linux; keep the lazy
+     * setting everywhere else, where commit is NOT free and the upfront-memory
+     * reason for it still holds (Windows especially — see #581). */
+#if defined(__linux__)
+    mem_option_set_verified(mi_option_arena_eager_commit, 2, "arena_eager_commit");
+#else
     mem_option_set_verified(mi_option_arena_eager_commit, 0, "arena_eager_commit");
+#endif
     mem_option_set_verified(mi_option_purge_decommits, SKIP_ONE, "purge_decommits");
     mem_option_set_verified(mi_option_purge_delay, 0, "purge_delay"); /* immediate */
     /* v3 (#832): reclaim abandoned pages on ANY thread's free (=1), restoring the

tests/test_mem.c

@@ -41,6 +41,32 @@
 
 /* ── mem basic tests ──────────────────────────────────────────── */
 
+/* Since #1360 routed ordinary malloc/new through mimalloc on Linux, the arena
+ * policy governs EVERY allocation in the process, not just the bound
+ * sqlite/tree_sitter populations. With lazy arena commit (0), mimalloc commits
+ * sub-ranges via mprotect(PROT_READ|PROT_WRITE) over a PROT_NONE reservation,
+ * and each partial commit SPLITS the reserved VMA: an index worker on the Go
+ * corpus held ~22k mappings where v0.9.0 held 10, growing with worker count.
+ * That is how #1654's 96-CPU host reached vm.max_map_count, after which mmap
+ * fails for ANY size — 10 KB allocations failing while `free -g` still showed
+ * 246 GB available. mimalloc's own default is 2, meaning "eager-commit arenas
+ * only on an OS that overcommits (i.e. linux)", where commit is free until the
+ * pages are touched; overriding it to 0 opted Linux out of the default written
+ * for Linux. Measured: 22450 -> 17312 mappings, wall time and peak RSS
+ * unchanged. Pin the platform split so the Linux default cannot be silently
+ * opted out again — and keep the lazy setting where commit is NOT free
+ * (Windows especially, see #581). */
+TEST(mem_arena_eager_commit_follows_platform_commit_cost) {
+    cbm_mem_init(0.5);
+    long eager = mi_option_get(mi_option_arena_eager_commit);
+#if defined(__linux__)
+    ASSERT_EQ(eager, 2);
+#else
+    ASSERT_EQ(eager, 0);
+#endif
+    PASS();
+}
+
 TEST(mem_rss_tracking) {
     cbm_mem_init(0.5);
 
@@ -1253,6 +1279,7 @@ TEST(mem_map_attributes_a_known_allocation) {
 
 SUITE(mem) {
     /* mem API */
+    RUN_TEST(mem_arena_eager_commit_follows_platform_commit_cost);
     RUN_TEST(mem_map_attributes_a_known_allocation);
     RUN_TEST(mem_rss_tracking);
     RUN_TEST(mem_collect_reclaims);