doris/post-build.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

##############################################################
# Post-build script for packaging third-party filesystem JARs
# (JuiceFS, JindoFS) and the Lance JNI library into Doris output directories.
#
# This script is independent of the main build and can be
# executed standalone or called from build.sh.
#
# Usage:
#   post-build.sh [--fe] [--be] [--output <dir>]
#                 [--juicefs] [--jindofs]
#
# Environment variables:
#   DISABLE_BUILD_JUICEFS=OFF  Enable JuiceFS packaging (default: ON, i.e. skip)
#   DISABLE_BUILD_JINDOFS=OFF  Enable JindoFS packaging (default: ON, i.e. skip)
##############################################################

set -eo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

DORIS_HOME="${DORIS_HOME:-${ROOT}}"
if [[ -z "${DORIS_THIRDPARTY}" ]]; then
    DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
fi

TARGET_SYSTEM="${TARGET_SYSTEM:-$(uname -s)}"
TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"

# --- Source helper scripts ---
. "${DORIS_HOME}/docker/thirdparties/juicefs-helpers.sh"
. "${DORIS_HOME}/docker/thirdparties/jindofs-helpers.sh"
. "${DORIS_HOME}/docker/thirdparties/lance-jni-helpers.sh"

# --- JuiceFS wrapper functions (migrated from build.sh) ---

find_juicefs_hadoop_jar() {
    juicefs_find_hadoop_jar_by_globs \
        "${DORIS_THIRDPARTY}/installed/juicefs_libs/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_THIRDPARTY}/src/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_HOME}/thirdparty/installed/juicefs_libs/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_HOME}/thirdparty/src/juicefs-hadoop-[0-9]*.jar"
}

detect_juicefs_version() {
    local juicefs_jar
    juicefs_jar=$(find_juicefs_hadoop_jar || true)
    juicefs_detect_hadoop_version "${juicefs_jar}" "${JUICEFS_DEFAULT_VERSION}"
}

download_juicefs_hadoop_jar() {
    local juicefs_version="$1"
    local cache_dir="${DORIS_HOME}/thirdparty/installed/juicefs_libs"
    juicefs_download_hadoop_jar_to_cache "${juicefs_version}" "${cache_dir}"
}

copy_juicefs_hadoop_jar() {
    local target_dir="$1"
    local source_jar=""
    source_jar=$(find_juicefs_hadoop_jar || true)
    if [[ -z "${source_jar}" ]]; then
        local juicefs_version
        juicefs_version=$(detect_juicefs_version)
        source_jar=$(download_juicefs_hadoop_jar "${juicefs_version}" || true)
    fi
    if [[ -z "${source_jar}" ]]; then
        echo "WARN: skip copying juicefs-hadoop jar, not found in thirdparty and download failed"
        return 0
    fi
    cp -r -p "${source_jar}" "${target_dir}/"
    echo "Copy JuiceFS Hadoop jar to ${target_dir}: $(basename "${source_jar}")"
}

# --- Install functions ---

install_juicefs() {
    local target_type="$1"
    local output_dir="$2"

    local target_dir
    if [[ "${target_type}" == "fe" ]]; then
        target_dir="${output_dir}/fe/lib/juicefs"
    elif [[ "${target_type}" == "be" ]]; then
        target_dir="${output_dir}/be/lib/java_extensions/juicefs"
    else
        echo "ERROR: unknown target type '${target_type}' for install_juicefs"
        return 1
    fi

    install -d "${target_dir}"
    copy_juicefs_hadoop_jar "${target_dir}"
}

install_jindofs() {
    local target_type="$1"
    local output_dir="$2"

    local target_dir
    if [[ "${target_type}" == "fe" ]]; then
        target_dir="${output_dir}/fe/lib/jindofs"
    elif [[ "${target_type}" == "be" ]]; then
        target_dir="${output_dir}/be/lib/java_extensions/jindofs"
    else
        echo "ERROR: unknown target type '${target_type}' for install_jindofs"
        return 1
    fi

    install -d "${target_dir}"
    jindofs_copy_jars "${DORIS_THIRDPARTY}/installed/jindofs_libs" "${target_dir}" \
        "${TARGET_SYSTEM}" "${TARGET_ARCH}"
}

# --- CLI ---

usage() {
    echo "
Usage: $0 [--fe] [--be] [--output <dir>] [--juicefs] [--jindofs]

Package third-party filesystem JARs (JuiceFS, JindoFS) into Doris output directories.
Linux x86_64 FE packaging automatically installs the glibc 2.17 Lance JNI library.
Use --juicefs/--jindofs or environment variables to opt into filesystem JAR packaging.

Options:
  --fe               Process FE output directory
  --be               Process BE output directory
  --output <dir>     Output directory (default: \${DORIS_HOME}/output)
  --juicefs          Enable JuiceFS packaging
  --jindofs          Enable JindoFS packaging

Environment variables:
  DISABLE_BUILD_JUICEFS=OFF  Enable JuiceFS packaging (--juicefs takes precedence)
  DISABLE_BUILD_JINDOFS=OFF  Enable JindoFS packaging (--jindofs takes precedence)
"
}

# Defaults: not installed
PROCESS_FE=0
PROCESS_BE=0
OUTPUT_DIR=""
BUILD_JUICEFS=""
BUILD_JINDOFS=""

while [[ $# -gt 0 ]]; do
    case "$1" in
        --fe)
            PROCESS_FE=1
            shift
            ;;
        --be)
            PROCESS_BE=1
            shift
            ;;
        --output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        --juicefs)
            BUILD_JUICEFS="ON"
            shift
            ;;
        --jindofs)
            BUILD_JINDOFS="ON"
            shift
            ;;
        --help | -h)
            usage
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            usage
            exit 1
            ;;
    esac
done

OUTPUT_DIR="${OUTPUT_DIR:-${DORIS_HOME}/output}"

# Resolve build flags: CLI > env DISABLE_BUILD_XXX > default (OFF)
if [[ -z "${BUILD_JUICEFS}" ]]; then
    if [[ "$(echo "${DISABLE_BUILD_JUICEFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then
        BUILD_JUICEFS="ON"
    else
        BUILD_JUICEFS="OFF"
    fi
fi
if [[ -z "${BUILD_JINDOFS}" ]]; then
    if [[ "$(echo "${DISABLE_BUILD_JINDOFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then
        BUILD_JINDOFS="ON"
    else
        BUILD_JINDOFS="OFF"
    fi
fi

if [[ "${PROCESS_FE}" -eq 0 && "${PROCESS_BE}" -eq 0 ]]; then
    echo "Nothing to do. Specify --fe and/or --be."
    exit 0
fi

# --- Main ---

for target_type in fe be; do
    if [[ "${target_type}" == "fe" && "${PROCESS_FE}" -eq 0 ]]; then
        continue
    fi
    if [[ "${target_type}" == "be" && "${PROCESS_BE}" -eq 0 ]]; then
        continue
    fi

    if [[ "${target_type}" == "fe" ]]; then
        lance_jni_replace "${OUTPUT_DIR}" "${DORIS_THIRDPARTY}" "${TARGET_SYSTEM}" "${TARGET_ARCH}"
    fi

    if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
        install_jindofs "${target_type}" "${OUTPUT_DIR}"
    fi

    if [[ "${BUILD_JUICEFS}" == "ON" ]]; then
        install_juicefs "${target_type}" "${OUTPUT_DIR}"
    fi
done