doris/run-cloud-ut.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

#####################################################################
# This script is used to run unit test of Doris Backend
# Usage: $0 <options>
#  Optional options:
#     --clean            clean and build ut
#     --run              build and run all ut
#     --run --filter=xx  build and run specified ut
#     -j                 build parallel
#     -h                 print this help message
#
# All BE tests must use "_test" as the file suffix, and add the file
# to cloud/test/CMakeLists.txt.
#
# GTest result xml files will be in "cloud/ut_build_ASAN/gtest_output/"
#####################################################################

ROOT=$(dirname "$0")
ROOT=$(
    cd "${ROOT}"
    pwd
)

export DORIS_HOME="${ROOT}"

trim_whitespace() {
    local value="$1"
    value="${value#"${value%%[![:space:]]*}"}"
    value="${value%"${value##*[![:space:]]}"}"
    printf '%s' "${value}"
}

is_valid_extra_module_feature() {
    local feature="$1"
    [[ "${feature}" =~ ^[A-Za-z][A-Za-z0-9_-]*$ ]]
}

feature_to_cmake_name() {
    local feature="$1"
    printf '%s' "${feature}" | tr '[:lower:]-' '[:upper:]_'
}

parse_extra_cloud_modules() {
    local spec_value="$1"
    local entry feature module_path existing
    local -a feature_keys=()

    CLOUD_EXTRA_FEATURE_KEYS=()
    CLOUD_EXTRA_MODULE_PATHS=()
    if [[ -z "${spec_value}" ]]; then
        return
    fi

    IFS=',' read -r -a entries <<<"${spec_value}"
    for entry in "${entries[@]}"; do
        entry="$(trim_whitespace "${entry}")"
        if [[ -z "${entry}" || "${entry}" != *=* ]]; then
            echo "Invalid EXTRA_CLOUD_MODULES entry '${entry}': expected feature=module_path"
            exit 1
        fi

        feature="$(trim_whitespace "${entry%%=*}")"
        module_path="$(trim_whitespace "${entry#*=}")"
        if [[ -z "${feature}" || -z "${module_path}" ]]; then
            echo "Invalid EXTRA_CLOUD_MODULES entry '${entry}': feature and module_path must be non-empty"
            exit 1
        fi
        if ! is_valid_extra_module_feature "${feature}"; then
            echo "Invalid EXTRA_CLOUD_MODULES feature '${feature}'"
            exit 1
        fi
        for existing in "${feature_keys[@]}"; do
            if [[ "${existing}" == "${feature}" ]]; then
                echo "Duplicate EXTRA_CLOUD_MODULES feature '${feature}'"
                exit 1
            fi
        done
        if [[ ! -d "${DORIS_HOME}/cloud/src/${module_path}" ]]; then
            echo "Missing EXTRA_CLOUD_MODULES module directory: ${DORIS_HOME}/cloud/src/${module_path}"
            exit 1
        fi
        feature_keys+=("${feature}")
        CLOUD_EXTRA_FEATURE_KEYS+=("${feature}")
        CLOUD_EXTRA_MODULE_PATHS+=("${module_path}")
    done
}

# Check args
usage() {
    echo "
Usage: $0 <options>
  Optional options:
     --clean            clean and build ut
     --run              build and run all ut
     --coverage         coverage after run ut, does not take if --gdb is specified
     --run --filter=x   build and run specified ut, filter x format is <binary_name>:<gtest_filter>,
                        a <binary_name> is the name of a cpp file without '.cpp' suffix.
                        e.g. binary_name of xxx_test.cpp is xxx_test
     --fdb              run with a specific fdb connection string, e.g fdb_cluster0:cluster0@192.168.1.100:4500
     -j                 build parallel
     -h                 print this help message
     --gdb              debug with gdb, does not take effect if --run is not specified

  Environment variables:
     EXTRA_CLOUD_MODULES Optional CLOUD feature modules in feature=module_path format, separated by commas.

  Eg.
    $0                                                                          build tests
    $0 --run                                                                    build and run all tests
    $0 --run --filter=recycler_test:FooTest.*                                   runs everying of test suite FooTest in recycler_test.cpp
    $0 --run --filter=recycler_test:*Null*:*Constructor*                        runs any test whose full name contains either 'Null' or 'Constructor' in recycler_test.cpp
    $0 --run --filter=recycler_test:-*DeathTest.*                               runs all non-death tests in recycler_test.cpp
    $0 --run --filter=recycler_test:FooTest.*-FooTest.Bar                       runs everything in test suite FooTest except FooTest.Bar in recycler_test.cpp
    $0 --run --filter=recycler_test:FooTest.*:BarTest.*-FooTest.Bar:BarTest.Foo runs everything in test suite FooTest except FooTest.Bar and everything in test suite BarTest except BarTest.Foo in recycler_test.cpp
    $0 --run --fdb=fdb_cluster0:cluster0@192.168.1.100:4500                     run with specific fdb
    $0 --run --coverage                                                         run with coverage report
    $0 --clean                                                                  clean and build tests
    $0 --clean --run                                                            clean, build and run all tests
    $0 --clean --run --gdb --filter=recycler_test:FooTest.*-FooTest.Bar        clean, build, run all tests and debug FooTest with gdb
    "
    exit 1
}

if ! OPTS=$(getopt -n "$0" -o vhj:f: -l run,gdb,clean,filter:,fdb:,coverage -- "$@"); then
    usage
fi

set -eo pipefail

eval set -- "${OPTS}"

PARALLEL=$(($(nproc) / 5 + 1))

CLEAN=0
RUN=0
FILTER=""
FDB=""
GDB=""
COVERAGE=""
ENABLE_CLANG_COVERAGE=OFF
BUILD_AZURE="ON"

echo "===================== filter: ${FILTER}"
if [[ $# != 1 ]]; then
    while true; do
        case "$1" in
        --clean)
            CLEAN=1
            shift
            ;;
        --run)
            RUN=1
            shift
            ;;
        --gdb)
            GDB="--gdb"
            shift
            ;;
        --coverage)
            ENABLE_CLANG_COVERAGE="ON"
            COVERAGE="--coverage"
            shift
            ;;
        --fdb)
            FDB="$2"
            shift 2
            ;;
        -f | --filter)
            FILTER="$2"
            shift 2
            ;;
        -j)
            PARALLEL=$2
            shift 2
            ;;
        --)
            shift
            break
            ;;
        *)
            usage
            ;;
        esac
    done
fi

echo "===================== filter: ${FILTER}"

CMAKE_BUILD_TYPE=${BUILD_TYPE:-ASAN}
CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE^^}"
EXTRA_CLOUD_MODULES="${EXTRA_CLOUD_MODULES:-}"
parse_extra_cloud_modules "${EXTRA_CLOUD_MODULES}"

CLOUD_EXTRA_CMAKE_ARGS=()
for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do
    feature_name="$(feature_to_cmake_name "${CLOUD_EXTRA_FEATURE_KEYS[i]}")"
    CLOUD_EXTRA_CMAKE_ARGS+=("-DENABLE_${feature_name}=ON")
    CLOUD_EXTRA_CMAKE_ARGS+=("-D${feature_name}_MODULE_DIR=${CLOUD_EXTRA_MODULE_PATHS[i]}")
done

echo "Get params:
    PARALLEL            -- ${PARALLEL}
    CLEAN               -- ${CLEAN}
    FILTER              -- ${FILTER}
    COVERAGE            -- ${ENABLE_CLANG_COVERAGE}
    FDB                 -- ${FDB}
    EXTRA_CLOUD_MODULES -- ${EXTRA_CLOUD_MODULES}
"
echo "Build Doris Cloud UT"

if [[ "_${ENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
    sed -i "s/DORIS_TOOLCHAIN=gcc/DORIS_TOOLCHAIN=clang/g" env.sh
    echo "export DORIS_TOOLCHAIN=clang" >>custom_env.sh
fi

. "${DORIS_HOME}/env.sh"

CMAKE_BUILD_DIR="${DORIS_HOME}/cloud/ut_build_${CMAKE_BUILD_TYPE}"
if [[ "${CLEAN}" -eq 1 ]]; then
    rm "${CMAKE_BUILD_DIR}" -rf
    rm "${DORIS_HOME}/cloud/output/" -rf
fi

if [[ ! -d "${CMAKE_BUILD_DIR}" ]]; then
    mkdir -p "${CMAKE_BUILD_DIR}"
fi

if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
    GLIBC_COMPATIBILITY=ON
fi

if [[ -z "${USE_LIBCPP}" ]]; then
    if [[ "$(uname -s)" != 'Darwin' ]]; then
        USE_LIBCPP='OFF'
    else
        USE_LIBCPP='ON'
    fi
fi

if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
    BUILD_AZURE='OFF'
fi

MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")"
echo "-- Make program: ${MAKE_PROGRAM}"

cd "${CMAKE_BUILD_DIR}"
find . -name "*.gcda" -exec rm {} \;

"${CMAKE_CMD}" -G "${GENERATOR}" \
    -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
    -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
    -DMAKE_TEST=ON \
    -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
    -DUSE_LIBCPP="${USE_LIBCPP}" \
    -DENABLE_HDFS_STORAGE_VAULT=${ENABLE_HDFS_STORAGE_VAULT:-ON} \
    -DUSE_MEM_TRACKER=ON \
    -DUSE_JEMALLOC=OFF \
    -DSTRICT_MEMORY_USE=OFF \
    -DENABLE_CLANG_COVERAGE="${ENABLE_CLANG_COVERAGE}" \
    "${CMAKE_USE_CCACHE}" \
    -DBUILD_AZURE="${BUILD_AZURE}" \
    "${CLOUD_EXTRA_CMAKE_ARGS[@]}" \
    "${DORIS_HOME}/cloud/"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install

mkdir -p "${CMAKE_BUILD_DIR}/test/log"

# prepare java jars
LIB_DIR="${CMAKE_BUILD_DIR}/test/lib"
rm -rf "${LIB_DIR}"
mkdir "${LIB_DIR}"
if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then
    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs"
fi

if [[ "${RUN}" -ne 1 ]]; then
    echo "Finished"
    exit 0
fi

echo "**********************************"
echo "   Running MetaService Unit Test  "
echo "**********************************"

export ASAN_OPTIONS=detect_container_overflow=0

# test binary output dir
cd test
# FILTER: binary_name:gtest_filter
# FILTER: meta_service_test:DetachSchemaKVTest.*
# ./run_all_tests.sh --test "\"$(echo "${FILTER}" | awk -F: '{print $1}')\"" --filter "\"$(echo "${FILTER}" | awk -F: '{print $2}')\"" --fdb "\"${FDB}\""
set -euo pipefail
bash -x ./run_all_tests.sh "${COVERAGE}" --test "$(echo "${FILTER}" | awk -F: '{print $1}')" --filter "$(echo "${FILTER}" | awk -F: '{print $2}')" --fdb "${FDB}" "${GDB}"