From b65a564ea5bc6228977ef46a09085359be7bcc66 Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 4 Nov 2026 10:00:12 +0800 Subject: [PATCH] [fix](be) Route glibc allocation entry points through jemalloc (#69819) ### What problem does this PR solve? Problem Summary: On glibc < 2.25, dynamic TLS can be allocated by libc and freed by jemalloc, crashing during thread cleanup. Route all seven glibc allocation entry points through the existing jemalloc wrappers, preserve allocator API semantics, and add native interposition tests. ### Release note Fix a BE crash during dynamic TLS cleanup on older glibc systems and keep glibc allocation entry points paired with jemalloc. ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason `./run-be-ut.sh --jemalloc-hook --run -j 2`: 12/12 passed on Linux x86_64/glibc 2.32. Full BE, older glibc, ARM64 and customer JNI E2E were not run. - Behavior changed: - [ ] No. - [x] Yes. Glibc allocation entry points now consistently use jemalloc, with allocator boundary semantics preserved. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/runtime/memory/jemalloc_hook.cpp | 41 ++- be/test/jemalloc_hook/CMakeLists.txt | 58 ++++ be/test/jemalloc_hook/jemalloc_hook_test.cpp | 300 ++++++++++++++++++ .../jemalloc_hook/jemalloc_hook_test_dso.cpp | 44 +++ run-be-ut.sh | 32 +- 5 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 be/test/jemalloc_hook/CMakeLists.txt create mode 100644 be/test/jemalloc_hook/jemalloc_hook_test.cpp create mode 100644 be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp diff --git a/be/src/runtime/memory/jemalloc_hook.cpp b/be/src/runtime/memory/jemalloc_hook.cpp index 9cef99757bf..a70abd1f1e1 100644 --- a/be/src/runtime/memory/jemalloc_hook.cpp +++ b/be/src/runtime/memory/jemalloc_hook.cpp @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +#include + +#include +#include + #include "jemalloc/jemalloc.h" #ifndef __THROW @@ -35,7 +40,13 @@ void doris_free(void* p) __THROW { } void* doris_realloc(void* p, size_t size) __THROW { - return jerealloc(p, size); + void* ptr = jerealloc(p, size); + // jemalloc 5.3.0 can reject an oversized request without setting errno. + // realloc(p, 0) may free p and return nullptr, so only nonzero sizes fail. + if (ptr == nullptr && size != 0) { + errno = ENOMEM; + } + return ptr; } void* doris_calloc(size_t n, size_t size) __THROW { @@ -55,11 +66,21 @@ void* doris_aligned_alloc(size_t align, size_t size) __THROW { } void* doris_valloc(size_t size) __THROW { - return jevalloc(size); + // Unlike jemalloc's legacy valloc, aligned_alloc keeps zero-size results + // page-aligned and sets errno when an allocation fails. + return jealigned_alloc(getpagesize(), size); } void* doris_pvalloc(size_t size) __THROW { - return jevalloc(size); + const size_t page_size = getpagesize(); + // pvalloc rounds up to whole pages; SIZE_MAX must fail rather than wrap + // to a small allocation when adding the page-size adjustment. + if (size > std::numeric_limits::max() - (page_size - 1)) { + errno = ENOMEM; + return nullptr; + } + const size_t rounded_size = (size + page_size - 1) / page_size * page_size; + return jealigned_alloc(page_size, rounded_size); } int doris_posix_memalign(void** r, size_t align, size_t size) __THROW { @@ -77,9 +98,21 @@ void* realloc(void* p, size_t size) __THROW ALIAS(doris_realloc); void* calloc(size_t n, size_t size) __THROW ALIAS(doris_calloc); void cfree(void* ptr) __THROW ALIAS(doris_cfree); void* memalign(size_t align, size_t size) __THROW ALIAS(doris_memalign); +#ifdef __GLIBC__ +// Keep glibc's exported allocation entry points paired with the public jemalloc +// hooks. For example, before glibc 2.25, dlopened modules allocate dynamic TLS +// with __libc_memalign and release it through free. +void* __libc_malloc(size_t size) __THROW ALIAS(doris_malloc); +void __libc_free(void* ptr) __THROW ALIAS(doris_free); +void* __libc_calloc(size_t n, size_t size) __THROW ALIAS(doris_calloc); +void* __libc_realloc(void* ptr, size_t size) __THROW ALIAS(doris_realloc); +void* __libc_memalign(size_t align, size_t size) __THROW ALIAS(doris_memalign); +void* __libc_valloc(size_t size) __THROW ALIAS(doris_valloc); +void* __libc_pvalloc(size_t size) __THROW ALIAS(doris_pvalloc); +#endif void* aligned_alloc(size_t align, size_t size) __THROW ALIAS(doris_aligned_alloc); void* valloc(size_t size) __THROW ALIAS(doris_valloc); void* pvalloc(size_t size) __THROW ALIAS(doris_pvalloc); int posix_memalign(void** r, size_t a, size_t s) __THROW ALIAS(doris_posix_memalign); size_t malloc_usable_size(void* ptr) __THROW ALIAS(doris_malloc_usable_size); -} \ No newline at end of file +} diff --git a/be/test/jemalloc_hook/CMakeLists.txt b/be/test/jemalloc_hook/CMakeLists.txt new file mode 100644 index 00000000000..201a4c68405 --- /dev/null +++ b/be/test/jemalloc_hook/CMakeLists.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This allocator test must not use ASAN: its interceptors would hide missing +# glibc allocation entry points. Keep it separate from doris_be_test, which +# run-be-ut.sh builds with USE_JEMALLOC=OFF. +cmake_minimum_required(VERSION 3.20) +project(doris_jemalloc_hook_test LANGUAGES CXX) + +if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "jemalloc hook tests require Linux/glibc") +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +get_filename_component(DORIS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed") +find_package(Threads REQUIRED) + +foreach (LIB_NAME gtest gtest_main jemalloc_doris unwind lzma z) + find_library(${LIB_NAME}_LIBRARY NAMES ${LIB_NAME} + PATHS "${THIRDPARTY_DIR}/lib" "${THIRDPARTY_DIR}/lib64" + NO_DEFAULT_PATH REQUIRED) +endforeach() + +add_library(jemalloc_hook_test_dso SHARED jemalloc_hook_test_dso.cpp) +target_compile_options(jemalloc_hook_test_dso PRIVATE -ftls-model=global-dynamic) + +add_executable(jemalloc_hook_test + jemalloc_hook_test.cpp + "${DORIS_SOURCE_DIR}/be/src/runtime/memory/jemalloc_hook.cpp") +target_include_directories(jemalloc_hook_test PRIVATE "${THIRDPARTY_DIR}/include") +target_compile_definitions(jemalloc_hook_test PRIVATE + JEMALLOC_HOOK_TEST_DSO="$") +# Do not enable ENABLE_EXPORTS: verify the hook also works with the production +# executable's default symbol visibility. +# libunwind's compressed .debug_frame reader needs zlib's uncompress. +# Keep its compression archives after unwind in the static link order. +target_link_libraries(jemalloc_hook_test PRIVATE + "${gtest_main_LIBRARY}" "${gtest_LIBRARY}" "${jemalloc_doris_LIBRARY}" + "${unwind_LIBRARY}" "${lzma_LIBRARY}" "${z_LIBRARY}" + Threads::Threads ${CMAKE_DL_LIBS}) +add_dependencies(jemalloc_hook_test jemalloc_hook_test_dso) diff --git a/be/test/jemalloc_hook/jemalloc_hook_test.cpp b/be/test/jemalloc_hook/jemalloc_hook_test.cpp new file mode 100644 index 00000000000..10eacad09f7 --- /dev/null +++ b/be/test/jemalloc_hook/jemalloc_hook_test.cpp @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" void* doris_memalign(size_t alignment, size_t size) noexcept; + +namespace doris { + +class JemallocHookTest : public testing::Test { +protected: + using AllocationFunction = void* (*)(size_t); + using CallocFunction = void* (*)(size_t, size_t); + using FreeFunction = void (*)(void*); + using ReallocationFunction = void* (*)(void*, size_t); + using MemalignFunction = void* (*)(size_t, size_t); + + void SetUp() override { + ASSERT_NO_FATAL_FAILURE(check_executable_interposition()); + ASSERT_NO_FATAL_FAILURE(check_legacy_allocation_pair()); + } + + void check_executable_interposition() { + // Anchor the lookup to the linked production hook. A libc allocation + // paired with a libc free could otherwise pass without testing the hook. + Dl_info hook_image = {}; + ASSERT_NE(0, dladdr(reinterpret_cast(&doris_memalign), &hook_image)); + constexpr std::array symbol_names = { + "__libc_memalign", "malloc", "free", + "__libc_malloc", "__libc_free", "__libc_calloc", + "__libc_realloc", "__libc_valloc", "__libc_pvalloc"}; + for (size_t i = 0; i < symbol_names.size(); ++i) { + _allocator_symbols[i] = dlsym(RTLD_DEFAULT, symbol_names[i]); + ASSERT_NE(nullptr, _allocator_symbols[i]) << symbol_names[i] << ": " << dlerror(); + Dl_info symbol_image = {}; + ASSERT_NE(0, dladdr(_allocator_symbols[i], &symbol_image)) << symbol_names[i]; + ASSERT_EQ(hook_image.dli_fbase, symbol_image.dli_fbase) + << symbol_names[i] << " resolved to " << symbol_image.dli_fname + << " instead of " << hook_image.dli_fname; + } + _memalign = reinterpret_cast(_allocator_symbols[0]); + _libc_malloc = reinterpret_cast(_allocator_symbols[3]); + _libc_free = reinterpret_cast(_allocator_symbols[4]); + _libc_calloc = reinterpret_cast(_allocator_symbols[5]); + _libc_realloc = reinterpret_cast(_allocator_symbols[6]); + _libc_valloc = reinterpret_cast(_allocator_symbols[7]); + _libc_pvalloc = reinterpret_cast(_allocator_symbols[8]); + } + + void check_legacy_allocation_pair() { + // Exercise the legacy allocation/free pair even on newer glibc versions + // whose loader no longer uses __libc_memalign for TLS. A subprocess turns + // an allocator-mismatch crash into a test failure instead of aborting UT. + ASSERT_EXIT( + { + void* ptr = _memalign(16, 32); + const int result = (ptr == nullptr); + std::free(ptr); + std::_Exit(result); + }, + testing::ExitedWithCode(0), ""); + } + + std::array _allocator_symbols = {}; + MemalignFunction _memalign = nullptr; + AllocationFunction _libc_malloc = nullptr; + FreeFunction _libc_free = nullptr; + CallocFunction _libc_calloc = nullptr; + ReallocationFunction _libc_realloc = nullptr; + AllocationFunction _libc_valloc = nullptr; + AllocationFunction _libc_pvalloc = nullptr; +}; + +TEST_F(JemallocHookTest, LibcAllocationsCanBeFreedThroughPublicEntry) { + for (size_t size : {0, 17, 4097}) { + void* ptr = _libc_malloc(size); + ASSERT_NE(nullptr, ptr); + std::memset(ptr, 0x5a, size); + std::free(ptr); + } +} + +TEST_F(JemallocHookTest, PublicAllocationsCanBeFreedThroughLibcEntry) { + _libc_free(nullptr); + for (size_t size : {0, 17, 4097}) { + std::unique_ptr ptr(std::malloc(size), &std::free); + ASSERT_NE(nullptr, ptr.get()); + std::memset(ptr.get(), 0x5a, size); + errno = EDOM; + _libc_free(ptr.release()); + const int free_errno = errno; + EXPECT_EQ(EDOM, free_errno); + } +} + +TEST_F(JemallocHookTest, LibcCallocInitializesMemoryAndRejectsOverflow) { + for (size_t count : {0, 17}) { + constexpr size_t element_size = 13; + void* ptr = _libc_calloc(count, element_size); + ASSERT_NE(nullptr, ptr); + for (size_t i = 0; i < count * element_size; ++i) { + EXPECT_EQ(0, static_cast(ptr)[i]); + } + std::free(ptr); + } + + errno = 0; + EXPECT_EQ(nullptr, _libc_calloc(std::numeric_limits::max(), 2)); + EXPECT_EQ(ENOMEM, errno); +} + +TEST_F(JemallocHookTest, ReallocCanCrossLibcAndPublicEntryPoints) { + struct AllocationPair { + AllocationFunction allocate; + ReallocationFunction reallocate; + FreeFunction release; + }; + const std::array pairs = {{{&std::malloc, _libc_realloc, &std::free}, + {_libc_malloc, &std::realloc, _libc_free}, + {_libc_malloc, _libc_realloc, &std::free}}}; + for (const auto& pair : pairs) { + void* ptr = pair.allocate(17); + ASSERT_NE(nullptr, ptr); + std::memset(ptr, 0x5a, 17); + void* resized = pair.reallocate(ptr, 1031); + ASSERT_NE(nullptr, resized); + for (size_t i = 0; i < 17; ++i) { + EXPECT_EQ(0x5a, static_cast(resized)[i]); + } + std::memset(resized, 0x5a, 1031); + pair.release(resized); + } +} + +TEST_F(JemallocHookTest, LibcReallocFailurePreservesTheOriginalAllocation) { + void* ptr = _libc_realloc(nullptr, 17); + ASSERT_NE(nullptr, ptr); + std::memset(ptr, 0x5a, 17); + + errno = 0; + void* resized = _libc_realloc(ptr, std::numeric_limits::max()); + const int allocation_errno = errno; + ASSERT_EQ(nullptr, resized); + EXPECT_EQ(ENOMEM, allocation_errno); + for (size_t i = 0; i < 17; ++i) { + EXPECT_EQ(0x5a, static_cast(ptr)[i]); + } + std::free(ptr); +} + +TEST_F(JemallocHookTest, LibcReallocZeroSizeIsNotReportedAsOutOfMemory) { + std::unique_ptr ptr(std::malloc(17), &std::free); + ASSERT_NE(nullptr, ptr.get()); + errno = 0; + void* resized = _libc_realloc(ptr.release(), 0); + const int allocation_errno = errno; + EXPECT_NE(ENOMEM, allocation_errno); + std::free(resized); +} + +TEST_F(JemallocHookTest, LibcPageAlignedAllocationsCanBeFreed) { + const long page_size = sysconf(_SC_PAGESIZE); + ASSERT_GT(page_size, 0); + for (auto allocate : {_libc_valloc, _libc_pvalloc}) { + for (size_t size : {size_t {0}, size_t {1}, static_cast(page_size) + 1}) { + void* ptr = allocate(size); + ASSERT_NE(nullptr, ptr); + EXPECT_EQ(0, reinterpret_cast(ptr) % page_size) << "size: " << size; + std::memset(ptr, 0x5a, size); + std::free(ptr); + } + errno = 0; + void* ptr = allocate(std::numeric_limits::max()); + const int allocation_errno = errno; + EXPECT_EQ(nullptr, ptr); + EXPECT_EQ(ENOMEM, allocation_errno); + } +} + +TEST_F(JemallocHookTest, LibcPvallocRoundsAllocationSizeToPages) { + const size_t page_size = getpagesize(); + for (size_t size : {size_t {0}, size_t {1}, page_size + 1}) { + void* ptr = _libc_pvalloc(size); + ASSERT_NE(nullptr, ptr); + const size_t rounded_size = (size + page_size - 1) / page_size * page_size; + ASSERT_GE(malloc_usable_size(ptr), rounded_size); + std::memset(ptr, 0x5a, rounded_size); + std::free(ptr); + } +} + +TEST_F(JemallocHookTest, AlignedAllocationsCanBeFreed) { + for (size_t alignment : {1, 8, 16, 64, 4096}) { + for (size_t size : {0, 1, 17, 4097}) { + void* ptr = _memalign(alignment, size); + ASSERT_NE(nullptr, ptr); + EXPECT_EQ(0, reinterpret_cast(ptr) % alignment); + std::memset(ptr, 0x5a, size); + std::free(ptr); + } + } +} + +TEST_F(JemallocHookTest, AllocationErrorsPreserveMemalignSemantics) { + errno = 0; + void* ptr = _memalign(3, 16); + EXPECT_EQ(nullptr, ptr); + EXPECT_EQ(EINVAL, errno); + std::free(ptr); + + errno = 0; + ptr = _memalign(64, std::numeric_limits::max()); + EXPECT_EQ(nullptr, ptr); + EXPECT_EQ(ENOMEM, errno); + std::free(ptr); +} + +TEST_F(JemallocHookTest, AllocationsCanBeFreedAfterAllocatingThreadsExit) { + std::array pointers = {}; + std::array threads; + for (size_t i = 0; i < threads.size(); ++i) { + threads[i] = std::thread([&, i] { pointers[i] = _memalign(64, 1031); }); + } + for (auto& thread : threads) { + thread.join(); + } + for (void* ptr : pointers) { + ASSERT_NE(nullptr, ptr); + std::memset(ptr, 0x5a, 1031); + std::free(ptr); + } +} + +TEST_F(JemallocHookTest, LateLoadedDynamicTlsCanBeFreedAfterThreadsExit) { + std::unique_ptr handle( + dlopen(JEMALLOC_HOOK_TEST_DSO, RTLD_NOW | RTLD_LOCAL), &dlclose); + ASSERT_NE(nullptr, handle) << dlerror(); + using GetAllocatorSymbols = void (*)(void**); + auto get_allocator_symbols = reinterpret_cast( + dlsym(handle.get(), "jemalloc_hook_test_allocator_symbols")); + ASSERT_NE(nullptr, get_allocator_symbols) << dlerror(); + std::array allocator_symbols = {}; + get_allocator_symbols(allocator_symbols.data()); + // Check the DSO's actual relocations for every glibc allocation entry point + // before exercising dynamic TLS. + ASSERT_EQ(_allocator_symbols, allocator_symbols); + + using TouchTlsFunction = int (*)(); + auto touch_tls = + reinterpret_cast(dlsym(handle.get(), "jemalloc_hook_test_touch_tls")); + ASSERT_NE(nullptr, touch_tls) << dlerror(); + + // Exceed glibc's default stack cache to exercise TLS deallocation instead + // of only keeping exited threads' stacks cached. + for (int round = 0; round < 8; ++round) { + std::array results = {}; + std::array threads; + for (size_t i = 0; i < threads.size(); ++i) { + threads[i] = std::thread([&, i] { + for (int call = 0; call < 8; ++call) { + results[i] = touch_tls(); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + for (int result : results) { + EXPECT_EQ(8, result); + } + } +} + +} // namespace doris diff --git a/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp b/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp new file mode 100644 index 00000000000..21486d3d66e --- /dev/null +++ b/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +extern "C" void* __libc_memalign(size_t alignment, size_t size) noexcept; +extern "C" void* __libc_malloc(size_t size) noexcept; +extern "C" void __libc_free(void* ptr) noexcept; +extern "C" void* __libc_calloc(size_t count, size_t size) noexcept; +extern "C" void* __libc_realloc(void* ptr, size_t size) noexcept; +extern "C" void* __libc_valloc(size_t size) noexcept; +extern "C" void* __libc_pvalloc(size_t size) noexcept; + +extern "C" void jemalloc_hook_test_allocator_symbols(void** symbols) { + symbols[0] = reinterpret_cast(&__libc_memalign); + symbols[1] = reinterpret_cast(&std::malloc); + symbols[2] = reinterpret_cast(&std::free); + symbols[3] = reinterpret_cast(&__libc_malloc); + symbols[4] = reinterpret_cast(&__libc_free); + symbols[5] = reinterpret_cast(&__libc_calloc); + symbols[6] = reinterpret_cast(&__libc_realloc); + symbols[7] = reinterpret_cast(&__libc_valloc); + symbols[8] = reinterpret_cast(&__libc_pvalloc); +} + +// Access TLS from a late-loaded library, as happens with JNI native libraries. +extern "C" int jemalloc_hook_test_touch_tls() { + static thread_local int tls_values[1024] = {}; + return ++tls_values[1023]; +} diff --git a/run-be-ut.sh b/run-be-ut.sh index 2485582ca6e..78e24ef9bdc 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -22,6 +22,7 @@ # Optional options: # --clean clean and build ut # --run build and run all ut +# --jemalloc-hook build standalone jemalloc hook tests without sanitizers # --run --filter=xx build and run specified ut # --gdb debug with gdb, does not take effect if --run is not specified # --coverage generate coverage report, does not take effect if --gdb is specified @@ -117,6 +118,7 @@ Usage: $0 Optional options: --clean clean and build ut --run build and run all ut + --jemalloc-hook build standalone Linux jemalloc hook tests without sanitizers --run --filter=xx build and run specified ut --run --gen_out generate expected check data for test --run --gen_regression_case generate regression test cases corrresponding to ut cases for ut cases that support it @@ -145,7 +147,7 @@ Usage: $0 exit 1 } -if ! OPTS="$(getopt -n "$0" -o vhj:f: -l gen_out,gen_regression_case,coverage,benchmark,run,gdb,clean,filter: -- "$@")"; then +if ! OPTS="$(getopt -n "$0" -o vhj:f: -l gen_out,gen_regression_case,coverage,benchmark,run,gdb,clean,jemalloc-hook,filter: -- "$@")"; then usage fi @@ -153,6 +155,7 @@ eval set -- "${OPTS}" CLEAN=0 RUN=0 +JEMALLOC_HOOK=0 GDB=0 DENABLE_CLANG_COVERAGE='OFF' BUILD_AZURE='ON' @@ -170,6 +173,10 @@ if [[ "$#" != 1 ]]; then RUN=1 shift ;; + --jemalloc-hook) + JEMALLOC_HOOK=1 + shift + ;; --gdb) GDB=1 shift @@ -209,6 +216,29 @@ if [[ -z "${PARALLEL}" ]]; then PARALLEL="$(($(nproc) / 5 + 1))" fi +# The regular BE UT uses ASAN and disables jemalloc. Exercise allocator +# interposition in a separate native executable, before initializing BE's +# unrelated submodules or configuring the sanitizer build. +if [[ "${JEMALLOC_HOOK}" -eq 1 ]]; then + # These modes belong to the sanitizer BE UT, not the native allocator test. + if [[ "${GDB}" -eq 1 || "${DENABLE_CLANG_COVERAGE}" == 'ON' || -n "${GEN_OUT}${GEN_REGRESSION_CASE}" ]]; then + echo "--jemalloc-hook supports --run, --filter, --clean and -j" + exit 1 + fi + JEMALLOC_TEST_BUILD_DIR="${DORIS_HOME}/be/ut_build_JEMALLOC" + if [[ "${CLEAN}" -eq 1 ]]; then + rm -rf "${JEMALLOC_TEST_BUILD_DIR}" + fi + "${CMAKE_CMD}" -S "${DORIS_HOME}/be/test/jemalloc_hook" \ + -B "${JEMALLOC_TEST_BUILD_DIR}" -G "${GENERATOR}" + "${CMAKE_CMD}" --build "${JEMALLOC_TEST_BUILD_DIR}" --parallel "${PARALLEL}" + if [[ "${RUN}" -eq 1 ]]; then + "${JEMALLOC_TEST_BUILD_DIR}/jemalloc_hook_test" "${FILTER:---gtest_filter=*}" \ + "--gtest_output=xml:${JEMALLOC_TEST_BUILD_DIR}/gtest_output/jemalloc_hook_test.xml" + fi + exit 0 +fi + CMAKE_BUILD_TYPE="${BUILD_TYPE_UT:-ASAN}" CMAKE_BUILD_TYPE="$(echo "${CMAKE_BUILD_TYPE}" | awk '{ print(toupper($0)) }')"