[enhance](Azure) Make azure dependency compile one config (#38239)

Previously the azure sdk would be compiled into BE by default. But
sometimes the BE would run in environment which only require AWS
s3-compatible functionality. So user can use `DISABLE_BUILD_AZURE` to
control whether build azure sdk into BE or not.

`export DISABLE_BUILD_AZURE=ON` to disable build with azure support

be/CMakeLists.txt

@@ -162,6 +162,14 @@ else()
     find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
 endif()
 
+# Set if use libazure or not
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+    add_definitions(-DUSE_AZURE)
+endif()
+
+
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
 
 include (cmake/thirdparty.cmake)

be/cmake/thirdparty.cmake

@@ -141,10 +141,12 @@ if (NOT OS_MACOSX)
     add_thirdparty(aws-s2n LIBNAME "lib/libs2n.a")
 endif()
 
-add_thirdparty(azure-core)
-add_thirdparty(azure-identity)
-add_thirdparty(azure-storage-blobs)
-add_thirdparty(azure-storage-common)
+if(BUILD_AZURE STREQUAL "ON")
+    add_thirdparty(azure-core)
+    add_thirdparty(azure-identity)
+    add_thirdparty(azure-storage-blobs)
+    add_thirdparty(azure-storage-common)
+endif()
 
 add_thirdparty(minizip LIB64)
 add_thirdparty(simdjson LIB64)

be/src/io/CMakeLists.txt

@@ -22,6 +22,9 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
 
 file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp)
+if(BUILD_AZURE STREQUAL "OFF")
+    list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/azure_obj_storage_client.cpp")
+endif()
 list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/benchmark/fs_benchmark_tool.cpp")
 
 add_library(IO STATIC ${IO_FILES})

be/src/util/s3_util.cpp

@@ -29,7 +29,9 @@
 #include <util/string_util.h>
 
 #include <atomic>
+#ifdef USE_AZURE
 #include <azure/storage/blobs/blob_container_client.hpp>
+#endif
 #include <cstdlib>
 #include <filesystem>
 #include <functional>
@@ -41,7 +43,9 @@
 #include "common/logging.h"
 #include "common/status.h"
 #include "cpp/sync_point.h"
+#ifdef USE_AZURE
 #include "io/fs/azure_obj_storage_client.h"
+#endif
 #include "io/fs/obj_storage_client.h"
 #include "io/fs/s3_obj_storage_client.h"
 #include "runtime/exec_env.h"
@@ -218,6 +222,7 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::create(const S3ClientConf
 
 std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_azure_client(
         const S3ClientConf& s3_conf) {
+#ifdef USE_AZURE
     auto cred =
             std::make_shared<Azure::Storage::StorageSharedKeyCredential>(s3_conf.ak, s3_conf.sk);
 
@@ -228,6 +233,10 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_azure_client(
     auto containerClient = std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(uri, cred);
     LOG_INFO("create one azure client with {}", s3_conf.to_string());
     return std::make_shared<io::AzureObjStorageClient>(std::move(containerClient));
+#else
+    LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building");
+    return nullptr;
+#endif
 }
 
 std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_s3_client(

be/test/io/fs/azure_test.cpp

@@ -18,10 +18,12 @@
 #include <fmt/core.h>
 #include <gtest/gtest.h>
 
+#ifdef USE_AZURE
 #include <azure/storage/blobs.hpp>
 #include <azure/storage/blobs/blob_client.hpp>
 #include <azure/storage/blobs/blob_container_client.hpp>
 #include <azure/storage/common/storage_credential.hpp>
+#endif
 #include <cstdio>
 #include <iostream>
 #include <stdexcept>
@@ -46,6 +48,7 @@ std::string GetConnectionString() {
 
 TEST(AzureTest, Write) {
     GTEST_SKIP() << "Skipping Azure test, because this test it to test the compile and linkage";
+#ifdef USE_AZURE
     using namespace Azure::Storage::Blobs;
 
     std::string accountName = config::test_s3_ak;
@@ -91,6 +94,7 @@ TEST(AzureTest, Write) {
     blobClient.DownloadTo(buffer.data(), buffer.size());
 
     std::cout << std::string(buffer.begin(), buffer.end()) << std::endl;
+#endif
 }
 
 } // namespace doris

build.sh

@@ -63,6 +63,7 @@ Usage: $0 <options>
     STRIP_DEBUG_INFO            If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Default is OFF.
     DISABLE_BE_JAVA_EXTENSIONS  If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF.
     DISABLE_JAVA_CHECK_STYLE    If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE.
+    DISABLE_BUILD_AZURE         If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE.
   Eg.
     $0                                      build all
     $0 --be                                 build Backend
@@ -158,6 +159,7 @@ HELP=0
 PARAMETER_COUNT="$#"
 PARAMETER_FLAG=0
 DENABLE_CLANG_COVERAGE='OFF'
+BUILD_AZURE='ON'
 BUILD_UI=1
 if [[ "$#" == 1 ]]; then
     # default
@@ -432,6 +434,10 @@ if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then
     DISABLE_JAVA_CHECK_STYLE='OFF'
 fi
 
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+    BUILD_AZURE='OFF'
+fi
+
 if [[ -z "${ENABLE_INJECTION_POINT}" ]]; then
     ENABLE_INJECTION_POINT='OFF'
 fi
@@ -587,6 +593,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
         -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
         -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
         -DDORIS_JAVA_HOME="${JAVA_HOME}" \
+        -DBUILD_AZURE="${BUILD_AZURE}" \
         "${DORIS_HOME}/be"
 
     if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
@@ -625,6 +632,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
         -DUSE_DWARF="${USE_DWARF}" \
         -DUSE_JEMALLOC="${USE_JEMALLOC}" \
         -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+        -DBUILD_AZURE="${BUILD_AZURE}" \
         -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
         "${DORIS_HOME}/cloud/"
     "${BUILD_SYSTEM}" -j "${PARALLEL}"

cloud/CMakeLists.txt

@@ -347,6 +347,12 @@ else ()
     set(MALLOCLIB tcmalloc)
 endif()
 
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+    add_definitions(-DUSE_AZURE)
+endif()
+
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     set(ASAN_LIBS -static-libasan)
     set(LSAN_LIBS -static-liblsan)

cloud/src/recycler/CMakeLists.txt

@@ -9,6 +9,10 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfdb_c -L${THIRDPARTY_DIR
 
 file(GLOB_RECURSE SRC_LIST CONFIGURE_DEPENDS *.cpp)
 
+if(BUILD_AZURE STREQUAL "OFF")
+    list(REMOVE_ITEM SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/azure_obj_client.cpp")
+endif()
+
 if(BUILD_CHECK_META STREQUAL "OFF")
     list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/meta_checker.cpp)
 endif ()

cloud/src/recycler/s3_accessor.cpp

@@ -25,8 +25,10 @@
 #include <gen_cpp/cloud.pb.h>
 
 #include <algorithm>
+#ifdef USE_AZURE
 #include <azure/storage/blobs/blob_container_client.hpp>
 #include <azure/storage/common/storage_credential.hpp>
+#endif
 #include <execution>
 #include <memory>
 #include <utility>
@@ -39,7 +41,9 @@
 #include "common/util.h"
 #include "cpp/obj_retry_strategy.h"
 #include "cpp/s3_rate_limiter.h"
+#ifdef USE_AZURE
 #include "recycler/azure_obj_client.h"
+#endif
 #include "recycler/obj_storage_client.h"
 #include "recycler/s3_obj_client.h"
 #include "recycler/storage_vault_accessor.h"
@@ -212,6 +216,7 @@ int S3Accessor::init() {
     });
     switch (conf_.provider) {
     case S3Conf::AZURE: {
+#ifdef USE_AZURE
         Azure::Storage::Blobs::BlobClientOptions options;
         options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
         options.Retry.MaxRetries = config::max_s3_client_retry;
@@ -231,6 +236,10 @@ int S3Accessor::init() {
         uri_ = uri_ + '/' + conf_.prefix;
         obj_client_ = std::make_shared<AzureObjClient>(std::move(container_client));
         return 0;
+#else
+        LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building");
+        return 0;
+#endif
     }
     default: {
         uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix;

common/cpp/obj_retry_strategy.cpp

@@ -36,7 +36,7 @@ bool S3CustomRetryStrategy::ShouldRetry(const Aws::Client::AWSError<Aws::Client:
     }
     return Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
 }
-
+#ifdef USE_AZURE
 AzureRetryRecordPolicy::AzureRetryRecordPolicy(int retry_cnt) : retry_cnt(retry_cnt) {}
 
 AzureRetryRecordPolicy::~AzureRetryRecordPolicy() = default;
@@ -58,4 +58,5 @@ std::unique_ptr<AzureRetryRecordPolicy::HttpPolicy> AzureRetryRecordPolicy::Clon
     ret->retry_cnt = 0;
     return ret;
 }
+#endif
 } // namespace doris

common/cpp/obj_retry_strategy.h

@@ -20,7 +20,9 @@
 #include <aws/core/client/AWSError.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 
+#ifdef USE_AZURE
 #include <azure/core/http/policies/policy.hpp>
+#endif
 
 namespace doris {
 class S3CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy {
@@ -32,6 +34,7 @@ public:
                      long attemptedRetries) const override;
 };
 
+#ifdef USE_AZURE
 class AzureRetryRecordPolicy final : public Azure::Core::Http::Policies::HttpPolicy {
 public:
     AzureRetryRecordPolicy(int retry_cnt);
@@ -45,4 +48,5 @@ public:
 private:
     mutable int retry_cnt;
 };
+#endif
 } // namespace doris

run-be-ut.sh

@@ -80,6 +80,7 @@ CLEAN=0
 RUN=0
 BUILD_BENCHMARK_TOOL='OFF'
 DENABLE_CLANG_COVERAGE='OFF'
+BUILD_AZURE='ON'
 FILTER=""
 if [[ "$#" != 1 ]]; then
     while true; do
@@ -163,6 +164,10 @@ if [[ "_${DENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
     echo "export DORIS_TOOLCHAIN=clang" >>custom_env.sh
 fi
 
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+    BUILD_AZURE='OFF'
+fi
+
 if [[ -z ${CMAKE_BUILD_DIR} ]]; then
     CMAKE_BUILD_DIR="${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}"
 fi
@@ -245,6 +250,7 @@ cd "${CMAKE_BUILD_DIR}"
     ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
     -DENABLE_PCH="${ENABLE_PCH}" \
     -DDORIS_JAVA_HOME="${JAVA_HOME}" \
+    -DBUILD_AZURE="${BUILD_AZURE}" \
     "${DORIS_HOME}/be"
 "${BUILD_SYSTEM}" -j "${PARALLEL}"
 

run-cloud-ut.sh

@@ -86,6 +86,8 @@ RUN=0
 FILTER=""
 FDB=""
 ENABLE_CLANG_COVERAGE=OFF
+BUILD_AZURE="ON"
+
 echo "===================== filter: ${FILTER}"
 if [[ $# != 1 ]]; then
     while true; do
@@ -164,6 +166,10 @@ if [[ -z "${USE_DWARF}" ]]; then
     USE_DWARF=OFF
 fi
 
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+    BUILD_AZURE='OFF'
+fi
+
 MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")"
 echo "-- Make program: ${MAKE_PROGRAM}"
 
@@ -182,6 +188,7 @@ find . -name "*.gcda" -exec rm {} \;
     -DSTRICT_MEMORY_USE=OFF \
     -DENABLE_CLANG_COVERAGE="${ENABLE_CLANG_COVERAGE}" \
     "${CMAKE_USE_CCACHE}" \
+    -DBUILD_AZURE="${BUILD_AZURE}" \
     "${DORIS_HOME}/cloud/"
 "${BUILD_SYSTEM}" -j "${PARALLEL}"
 "${BUILD_SYSTEM}" install

thirdparty/build-thirdparty.sh

@@ -73,6 +73,8 @@ else
     PARALLEL="$(($(nproc) / 4 + 1))"
 fi
 
+BUILD_AZURE="ON"
+
 while true; do
     case "$1" in
     -j)
@@ -119,6 +121,10 @@ if [[ "${HELP}" -eq 1 ]]; then
     usage
 fi
 
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+    BUILD_AZURE='OFF'
+fi
+
 echo "Get params:
     PARALLEL            -- ${PARALLEL}
     CLEAN               -- ${CLEAN}
@@ -1780,21 +1786,25 @@ build_base64() {
 
 # azure blob storage
 build_azure() {
-    check_if_source_exist "${AZURE_SOURCE}"
-    cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
-    azure_dir=$(pwd)
+    if [[ "${BUILD_AZURE}" == "OFF" ]]; then
+        echo "Skip build azure"
+    else
+        check_if_source_exist "${AZURE_SOURCE}"
+        cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
+        azure_dir=$(pwd)
 
-    rm -rf "${BUILD_DIR}"
-    mkdir -p "${BUILD_DIR}"
-    cd "${BUILD_DIR}"
+        rm -rf "${BUILD_DIR}"
+        mkdir -p "${BUILD_DIR}"
+        cd "${BUILD_DIR}"
 
-    # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports
-    AZURE_PORTS="vcpkg-custom-ports"
-    AZURE_MANIFEST_DIR="."
+        # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports
+        AZURE_PORTS="vcpkg-custom-ports"
+        AZURE_MANIFEST_DIR="."
 
-    "${CMAKE_CMD}" -G "${GENERATOR}" -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
-    "${BUILD_SYSTEM}" -j "${PARALLEL}"
-    "${BUILD_SYSTEM}" install
+        "${CMAKE_CMD}" -G "${GENERATOR}" -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
+        "${BUILD_SYSTEM}" -j "${PARALLEL}"
+        "${BUILD_SYSTEM}" install
+    fi
 }
 
 # dragonbox