perf(allocator): move code into cold path in Arena::alloc_layout (#21622)

Previously `Arena::alloc_layout` behaved as follows:

1. Call `try_alloc_layout_fast` (fast path for common case that allocation can be serviced by current chunk).
2. If that fails, call `alloc_layout_slow` (cold path).
3. Check that `alloc_layout_slow` succeeded, panic if not.

Step 3 did not add instructions to the fast path, but it was inlined into all call sites.

Since step 3 is only required if `try_alloc_layout_fast` fails (rare case), instead move step 3 into the cold path. This reduces the amount of code inlined into call sites, making those functions more likely to be eligible for inlining themselves.
This commit is contained in:
overlookmotel committed 2026-04-21 23:54:53 +00:00
1 parent 3cf7cef922
commit 1b58029300
1 file changed
+25 -5
+25 -5
View File
@@ -34,7 +34,13 @@ impl<const MIN_ALIGN: usize> Arena<MIN_ALIGN> {
/// Panics if reserving space matching `layout` fails.
#[inline(always)]
pub fn alloc_layout(&self, layout: Layout) -> NonNull<u8> {
self.try_alloc_layout(layout).unwrap_or_else(|_| oom())
let ptr =
self.try_alloc_layout_fast(layout).unwrap_or_else(|| self.alloc_layout_slow(layout));
#[cfg(all(feature = "track_allocations", not(feature = "disable_track_allocations")))]
self.stats.record_allocation();
ptr
}
/// Attempt to allocate space for an object with the given `Layout`.
@@ -51,7 +57,7 @@ impl<const MIN_ALIGN: usize> Arena<MIN_ALIGN> {
let res = if let Some(p) = self.try_alloc_layout_fast(layout) {
Ok(p)
} else {
self.alloc_layout_slow(layout).ok_or(AllocErr)
self.try_alloc_layout_slow(layout).ok_or(AllocErr)
};
#[cfg(all(feature = "track_allocations", not(feature = "disable_track_allocations")))]
@@ -303,11 +309,25 @@ impl<const MIN_ALIGN: usize> Arena<MIN_ALIGN> {
Some(new_ptr)
}
/// Slow path allocation for when we need to allocate a new chunk, because there isn't enough room
/// in our current chunk.
/// Slow path for `alloc_layout`.
/// Called when there isn't enough room in our current chunk, so need to allocate a new chunk.
#[inline(never)]
#[cold]
fn alloc_layout_slow(&self, layout: Layout) -> Option<NonNull<u8>> {
fn alloc_layout_slow(&self, layout: Layout) -> NonNull<u8> {
self.try_alloc_layout_slow_impl(layout).unwrap_or_else(|| oom())
}
/// Slow path for `try_alloc_layout`.
/// Called when there isn't enough room in our current chunk, so need to allocate a new chunk.
#[inline(never)]
#[cold]
fn try_alloc_layout_slow(&self, layout: Layout) -> Option<NonNull<u8>> {
self.try_alloc_layout_slow_impl(layout)
}
/// Slow path for allocation, shared between `alloc_layout` and `try_alloc_layout`.
/// Called when there isn't enough room in our current chunk, so need to allocate a new chunk.
fn try_alloc_layout_slow_impl(&self, layout: Layout) -> Option<NonNull<u8>> {
unsafe {
if !self.can_grow {
return None;