doris/post-build.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

##############################################################
# Post-build script for packaging third-party filesystem JARs
# (JuiceFS, JindoFS) into Doris FE/BE output directories.
#
# This script is independent of the main build and can be
# executed standalone or called from build.sh.
#
# Usage:
#   post-build.sh [--fe] [--be] [--output <dir>]
#                 [--juicefs] [--jindofs]
#
# Environment variables:
#   DISABLE_BUILD_JUICEFS=OFF  Enable JuiceFS packaging (default: ON, i.e. skip)
#   DISABLE_BUILD_JINDOFS=OFF  Enable JindoFS packaging (default: ON, i.e. skip)
##############################################################

set -eo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

DORIS_HOME="${DORIS_HOME:-${ROOT}}"
if [[ -z "${DORIS_THIRDPARTY}" ]]; then
    DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
fi

TARGET_SYSTEM="${TARGET_SYSTEM:-$(uname -s)}"
TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"

# --- Source helper scripts ---
. "${DORIS_HOME}/docker/thirdparties/juicefs-helpers.sh"
. "${DORIS_HOME}/docker/thirdparties/jindofs-helpers.sh"

# --- JuiceFS wrapper functions (migrated from build.sh) ---

find_juicefs_hadoop_jar() {
    juicefs_find_hadoop_jar_by_globs \
        "${DORIS_THIRDPARTY}/installed/juicefs_libs/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_THIRDPARTY}/src/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_HOME}/thirdparty/installed/juicefs_libs/juicefs-hadoop-[0-9]*.jar" \
        "${DORIS_HOME}/thirdparty/src/juicefs-hadoop-[0-9]*.jar"
}

detect_juicefs_version() {
    local juicefs_jar
    juicefs_jar=$(find_juicefs_hadoop_jar || true)
    juicefs_detect_hadoop_version "${juicefs_jar}" "${JUICEFS_DEFAULT_VERSION}"
}

download_juicefs_hadoop_jar() {
    local juicefs_version="$1"
    local cache_dir="${DORIS_HOME}/thirdparty/installed/juicefs_libs"
    juicefs_download_hadoop_jar_to_cache "${juicefs_version}" "${cache_dir}"
}

copy_juicefs_hadoop_jar() {
    local target_dir="$1"
    local source_jar=""
    source_jar=$(find_juicefs_hadoop_jar || true)
    if [[ -z "${source_jar}" ]]; then
        local juicefs_version
        juicefs_version=$(detect_juicefs_version)
        source_jar=$(download_juicefs_hadoop_jar "${juicefs_version}" || true)
    fi
    if [[ -z "${source_jar}" ]]; then
        echo "WARN: skip copying juicefs-hadoop jar, not found in thirdparty and download failed"
        return 0
    fi
    cp -r -p "${source_jar}" "${target_dir}/"
    echo "Copy JuiceFS Hadoop jar to ${target_dir}: $(basename "${source_jar}")"
}

# --- Install functions ---

# Where a third-party filesystem's jars are deployed, per target.
#
# FE keeps lib/<name>: it has exactly one consumer, the FE's own classpath.
#
# BE uses plugins/jni_fs/<name>, because it has TWO consumers and they need the same files. The
# native libhdfs reader wants them on the system class path (bin/start_be.sh puts them there), and
# every Java plugin wants them on its own classpath (PluginRegistry appends them, BE config
# jni_plugin_fs_dir) - a plugin classloader cannot reach the system class path by design, which is
# what took oss:// and jfs:// away from the Java scanners when the plugins were isolated. One
# directory read by both is one copy on disk; the JuiceFS Hadoop SDK alone is a 180 MB fat jar.
# It sits under plugins/ rather than lib/ for the same reason plugins/jni does: lib/ is the engine
# tree a package upgrade replaces wholesale.
fs_libs_dir() {
    local target_type="$1"
    local output_dir="$2"
    local name="$3"

    if [[ "${target_type}" == "fe" ]]; then
        echo "${output_dir}/fe/lib/${name}"
    elif [[ "${target_type}" == "be" ]]; then
        echo "${output_dir}/be/plugins/jni_fs/${name}"
    else
        return 1
    fi
}

install_juicefs() {
    local target_type="$1"
    local output_dir="$2"

    local target_dir
    if ! target_dir="$(fs_libs_dir "${target_type}" "${output_dir}" juicefs)"; then
        echo "ERROR: unknown target type '${target_type}' for install_juicefs"
        return 1
    fi

    install -d "${target_dir}"
    copy_juicefs_hadoop_jar "${target_dir}"
}

install_jindofs() {
    local target_type="$1"
    local output_dir="$2"

    local target_dir
    if ! target_dir="$(fs_libs_dir "${target_type}" "${output_dir}" jindofs)"; then
        echo "ERROR: unknown target type '${target_type}' for install_jindofs"
        return 1
    fi

    install -d "${target_dir}"
    jindofs_copy_jars "${DORIS_THIRDPARTY}/installed/jindofs_libs" "${target_dir}" \
        "${TARGET_SYSTEM}" "${TARGET_ARCH}"
}

# --- CLI ---

usage() {
    echo "
Usage: $0 [--fe] [--be] [--output <dir>] [--juicefs] [--jindofs]

Package third-party filesystem JARs (JuiceFS, JindoFS) into Doris output directories.
By default, no third-party JARs are packaged. Use --juicefs/--jindofs or environment variables to opt in.

Options:
  --fe               Process FE output directory
  --be               Process BE output directory
  --output <dir>     Output directory (default: \${DORIS_HOME}/output)
  --juicefs          Enable JuiceFS packaging
  --jindofs          Enable JindoFS packaging

Environment variables:
  DISABLE_BUILD_JUICEFS=OFF  Enable JuiceFS packaging (--juicefs takes precedence)
  DISABLE_BUILD_JINDOFS=OFF  Enable JindoFS packaging (--jindofs takes precedence)
"
}

# Defaults: not installed
PROCESS_FE=0
PROCESS_BE=0
OUTPUT_DIR=""
BUILD_JUICEFS=""
BUILD_JINDOFS=""

while [[ $# -gt 0 ]]; do
    case "$1" in
        --fe)
            PROCESS_FE=1
            shift
            ;;
        --be)
            PROCESS_BE=1
            shift
            ;;
        --output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        --juicefs)
            BUILD_JUICEFS="ON"
            shift
            ;;
        --jindofs)
            BUILD_JINDOFS="ON"
            shift
            ;;
        --help | -h)
            usage
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            usage
            exit 1
            ;;
    esac
done

OUTPUT_DIR="${OUTPUT_DIR:-${DORIS_HOME}/output}"

# Resolve build flags: CLI > env DISABLE_BUILD_XXX > default (OFF)
if [[ -z "${BUILD_JUICEFS}" ]]; then
    if [[ "$(echo "${DISABLE_BUILD_JUICEFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then
        BUILD_JUICEFS="ON"
    else
        BUILD_JUICEFS="OFF"
    fi
fi
if [[ -z "${BUILD_JINDOFS}" ]]; then
    if [[ "$(echo "${DISABLE_BUILD_JINDOFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then
        BUILD_JINDOFS="ON"
    else
        BUILD_JINDOFS="OFF"
    fi
fi

if [[ "${PROCESS_FE}" -eq 0 && "${PROCESS_BE}" -eq 0 ]]; then
    echo "Nothing to do. Specify --fe and/or --be."
    exit 0
fi

# --- Main ---

for target_type in fe be; do
    if [[ "${target_type}" == "fe" && "${PROCESS_FE}" -eq 0 ]]; then
        continue
    fi
    if [[ "${target_type}" == "be" && "${PROCESS_BE}" -eq 0 ]]; then
        continue
    fi

    if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
        install_jindofs "${target_type}" "${OUTPUT_DIR}"
    fi

    if [[ "${BUILD_JUICEFS}" == "ON" ]]; then
        install_juicefs "${target_type}" "${OUTPUT_DIR}"
    fi
done