#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ##################################################################### # This script is used to run unit test of Doris Backend # Usage: $0 # Optional options: # --clean clean and build ut # --run build and run all ut # --run --filter=xx build and run specified ut # -j build parallel # -h print this help message # # All BE tests must use "_test" as the file suffix, and add the file # to cloud/test/CMakeLists.txt. # # GTest result xml files will be in "cloud/ut_build_ASAN/gtest_output/" ##################################################################### ROOT=$(dirname "$0") ROOT=$( cd "${ROOT}" pwd ) export DORIS_HOME="${ROOT}" trim_whitespace() { local value="$1" value="${value#"${value%%[![:space:]]*}"}" value="${value%"${value##*[![:space:]]}"}" printf '%s' "${value}" } is_valid_extra_module_feature() { local feature="$1" [[ "${feature}" =~ ^[A-Za-z][A-Za-z0-9_-]*$ ]] } feature_to_cmake_name() { local feature="$1" printf '%s' "${feature}" | tr '[:lower:]-' '[:upper:]_' } parse_extra_cloud_modules() { local spec_value="$1" local entry feature module_path existing local -a feature_keys=() CLOUD_EXTRA_FEATURE_KEYS=() CLOUD_EXTRA_MODULE_PATHS=() if [[ -z "${spec_value}" ]]; then return fi IFS=',' read -r -a entries <<<"${spec_value}" for entry in "${entries[@]}"; do entry="$(trim_whitespace "${entry}")" if [[ -z "${entry}" || "${entry}" != *=* ]]; then echo "Invalid EXTRA_CLOUD_MODULES entry '${entry}': expected feature=module_path" exit 1 fi feature="$(trim_whitespace "${entry%%=*}")" module_path="$(trim_whitespace "${entry#*=}")" if [[ -z "${feature}" || -z "${module_path}" ]]; then echo "Invalid EXTRA_CLOUD_MODULES entry '${entry}': feature and module_path must be non-empty" exit 1 fi if ! is_valid_extra_module_feature "${feature}"; then echo "Invalid EXTRA_CLOUD_MODULES feature '${feature}'" exit 1 fi for existing in "${feature_keys[@]}"; do if [[ "${existing}" == "${feature}" ]]; then echo "Duplicate EXTRA_CLOUD_MODULES feature '${feature}'" exit 1 fi done if [[ ! -d "${DORIS_HOME}/cloud/src/${module_path}" ]]; then echo "Missing EXTRA_CLOUD_MODULES module directory: ${DORIS_HOME}/cloud/src/${module_path}" exit 1 fi feature_keys+=("${feature}") CLOUD_EXTRA_FEATURE_KEYS+=("${feature}") CLOUD_EXTRA_MODULE_PATHS+=("${module_path}") done } # Check args usage() { echo " Usage: $0 Optional options: --clean clean and build ut --run build and run all ut --coverage coverage after run ut, does not take if --gdb is specified --run --filter=x build and run specified ut, filter x format is :, a is the name of a cpp file without '.cpp' suffix. e.g. binary_name of xxx_test.cpp is xxx_test --fdb run with a specific fdb connection string, e.g fdb_cluster0:cluster0@192.168.1.100:4500 -j build parallel -h print this help message --gdb debug with gdb, does not take effect if --run is not specified Environment variables: EXTRA_CLOUD_MODULES Optional CLOUD feature modules in feature=module_path format, separated by commas. Eg. $0 build tests $0 --run build and run all tests $0 --run --filter=recycler_test:FooTest.* runs everying of test suite FooTest in recycler_test.cpp $0 --run --filter=recycler_test:*Null*:*Constructor* runs any test whose full name contains either 'Null' or 'Constructor' in recycler_test.cpp $0 --run --filter=recycler_test:-*DeathTest.* runs all non-death tests in recycler_test.cpp $0 --run --filter=recycler_test:FooTest.*-FooTest.Bar runs everything in test suite FooTest except FooTest.Bar in recycler_test.cpp $0 --run --filter=recycler_test:FooTest.*:BarTest.*-FooTest.Bar:BarTest.Foo runs everything in test suite FooTest except FooTest.Bar and everything in test suite BarTest except BarTest.Foo in recycler_test.cpp $0 --run --fdb=fdb_cluster0:cluster0@192.168.1.100:4500 run with specific fdb $0 --run --coverage run with coverage report $0 --clean clean and build tests $0 --clean --run clean, build and run all tests $0 --clean --run --gdb --filter=recycler_test:FooTest.*-FooTest.Bar clean, build, run all tests and debug FooTest with gdb " exit 1 } if ! OPTS=$(getopt -n "$0" -o vhj:f: -l run,gdb,clean,filter:,fdb:,coverage -- "$@"); then usage fi set -eo pipefail eval set -- "${OPTS}" PARALLEL=$(($(nproc) / 5 + 1)) CLEAN=0 RUN=0 FILTER="" FDB="" GDB="" COVERAGE="" ENABLE_CLANG_COVERAGE=OFF BUILD_AZURE="ON" echo "===================== filter: ${FILTER}" if [[ $# != 1 ]]; then while true; do case "$1" in --clean) CLEAN=1 shift ;; --run) RUN=1 shift ;; --gdb) GDB="--gdb" shift ;; --coverage) ENABLE_CLANG_COVERAGE="ON" COVERAGE="--coverage" shift ;; --fdb) FDB="$2" shift 2 ;; -f | --filter) FILTER="$2" shift 2 ;; -j) PARALLEL=$2 shift 2 ;; --) shift break ;; *) usage ;; esac done fi echo "===================== filter: ${FILTER}" CMAKE_BUILD_TYPE=${BUILD_TYPE:-ASAN} CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE^^}" EXTRA_CLOUD_MODULES="${EXTRA_CLOUD_MODULES:-}" parse_extra_cloud_modules "${EXTRA_CLOUD_MODULES}" CLOUD_EXTRA_CMAKE_ARGS=() for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do feature_name="$(feature_to_cmake_name "${CLOUD_EXTRA_FEATURE_KEYS[i]}")" CLOUD_EXTRA_CMAKE_ARGS+=("-DENABLE_${feature_name}=ON") CLOUD_EXTRA_CMAKE_ARGS+=("-D${feature_name}_MODULE_DIR=${CLOUD_EXTRA_MODULE_PATHS[i]}") done echo "Get params: PARALLEL -- ${PARALLEL} CLEAN -- ${CLEAN} FILTER -- ${FILTER} COVERAGE -- ${ENABLE_CLANG_COVERAGE} FDB -- ${FDB} EXTRA_CLOUD_MODULES -- ${EXTRA_CLOUD_MODULES} " echo "Build Doris Cloud UT" if [[ "_${ENABLE_CLANG_COVERAGE}" == "_ON" ]]; then sed -i "s/DORIS_TOOLCHAIN=gcc/DORIS_TOOLCHAIN=clang/g" env.sh echo "export DORIS_TOOLCHAIN=clang" >>custom_env.sh fi . "${DORIS_HOME}/env.sh" CMAKE_BUILD_DIR="${DORIS_HOME}/cloud/ut_build_${CMAKE_BUILD_TYPE}" if [[ "${CLEAN}" -eq 1 ]]; then rm "${CMAKE_BUILD_DIR}" -rf rm "${DORIS_HOME}/cloud/output/" -rf fi if [[ ! -d "${CMAKE_BUILD_DIR}" ]]; then mkdir -p "${CMAKE_BUILD_DIR}" fi if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then GLIBC_COMPATIBILITY=ON fi if [[ -z "${USE_LIBCPP}" ]]; then if [[ "$(uname -s)" != 'Darwin' ]]; then USE_LIBCPP='OFF' else USE_LIBCPP='ON' fi fi if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then BUILD_AZURE='OFF' fi MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")" echo "-- Make program: ${MAKE_PROGRAM}" cd "${CMAKE_BUILD_DIR}" find . -name "*.gcda" -exec rm {} \; "${CMAKE_CMD}" -G "${GENERATOR}" \ -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DMAKE_TEST=ON \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ -DUSE_LIBCPP="${USE_LIBCPP}" \ -DENABLE_HDFS_STORAGE_VAULT=${ENABLE_HDFS_STORAGE_VAULT:-ON} \ -DUSE_MEM_TRACKER=ON \ -DUSE_JEMALLOC=OFF \ -DSTRICT_MEMORY_USE=OFF \ -DENABLE_CLANG_COVERAGE="${ENABLE_CLANG_COVERAGE}" \ "${CMAKE_USE_CCACHE}" \ -DBUILD_AZURE="${BUILD_AZURE}" \ "${CLOUD_EXTRA_CMAKE_ARGS[@]}" \ "${DORIS_HOME}/cloud/" "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install mkdir -p "${CMAKE_BUILD_DIR}/test/log" # prepare java jars LIB_DIR="${CMAKE_BUILD_DIR}/test/lib" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ "${RUN}" -ne 1 ]]; then echo "Finished" exit 0 fi echo "**********************************" echo " Running MetaService Unit Test " echo "**********************************" export ASAN_OPTIONS=detect_container_overflow=0 # test binary output dir cd test # FILTER: binary_name:gtest_filter # FILTER: meta_service_test:DetachSchemaKVTest.* # ./run_all_tests.sh --test "\"$(echo "${FILTER}" | awk -F: '{print $1}')\"" --filter "\"$(echo "${FILTER}" | awk -F: '{print $2}')\"" --fdb "\"${FDB}\"" set -euo pipefail bash -x ./run_all_tests.sh "${COVERAGE}" --test "$(echo "${FILTER}" | awk -F: '{print $1}')" --filter "$(echo "${FILTER}" | awk -F: '{print $2}')" --fdb "${FDB}" "${GDB}"