# syntax=docker/dockerfile:1.4 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. # # BASE_IMAGE: the Docker Hardened Image that Python comes from. Both the build stage and the final # image are based on it, so Python is never compiled during the build. # # The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no # registry credentials. The upstream image itself requires a "docker login dhi.io". It is pinned to a # digest, because the mirror's tags move several times a day: "breeze ci upgrade" moves the pin once # the full CI has passed on the new image. # # The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the # default, but you can build a FIPS-compliant image by pointing the build at one: # docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.18-debian13-fips-dev" ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.18-debian13-dev@sha256:cacda18986eefebc42b0c6c03565ec2d65a4507b6adff559e4daf0f1e2bfe338" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments # We use BASE_IMAGE to make sure that the scripts are different for different platforms. ############################################################################################## FROM ${BASE_IMAGE} as scripts ############################################################################################## # Please DO NOT modify the inlined scripts manually. The content of those files will be # replaced by prek automatically from the "scripts/docker/" folder. # This is done in order to avoid problems with caching and file permissions and in order to # make the PROD Dockerfile standalone ############################################################################################## # The content below is automatically copied from scripts/docker/install_os_dependencies.sh COPY <<"EOF" /install_os_dependencies.sh #!/usr/bin/env bash set -euo pipefail if [[ "$#" != 1 ]]; then echo echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.". echo exit 1 fi GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4} TEMURIN_VERSION=${TEMURIN_VERSION:-11} NODEJS_VERSION=${NODEJS_VERSION:-22.23.1} PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as # Debian packages under /usr - so ask the base image's Python where it lives. PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} # Read before apt runs: a package depending on python3 can pull another Python version in. BASE_PYTHON_MAJOR_MINOR=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') if [[ -n "${AIRFLOW_PYTHON_VERSION}" && "${AIRFLOW_PYTHON_VERSION}." != "${BASE_PYTHON_MAJOR_MINOR}."* ]]; then echo echo "ERROR! AIRFLOW_PYTHON_VERSION is ${AIRFLOW_PYTHON_VERSION}, but the base image has Python ${BASE_PYTHON_MAJOR_MINOR}." echo "The hardened flavor takes Python from BASE_IMAGE: leave AIRFLOW_PYTHON_VERSION empty or pick a BASE_IMAGE with that Python." echo exit 1 fi elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else echo echo "ERROR! AIRFLOW_IMAGE_FLAVOR should be 'hardened' or 'legacy', not '${AIRFLOW_IMAGE_FLAVOR}'." echo exit 1 fi if [[ "${1}" == "runtime" ]]; then INSTALLATION_TYPE="RUNTIME" elif [[ "${1}" == "dev" ]]; then INSTALLATION_TYPE="DEV" elif [[ "${1}" == "ci" ]]; then INSTALLATION_TYPE="CI" else echo echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.". echo exit 1 fi function get_dev_apt_deps() { if [[ "${DEV_APT_DEPS=}" == "" ]]; then DEV_APT_DEPS="\ apt-transport-https \ apt-utils \ build-essential \ dirmngr \ freetds-bin \ freetds-dev \ git \ graphviz \ graphviz-dev \ gzip \ krb5-user \ ldap-utils \ libc6-dev \ libev-dev \ libev4 \ libffi-dev \ libgeos-dev \ libkrb5-dev \ libldap2-dev \ libleveldb-dev \ libleveldb1d \ libsasl2-2 \ libsasl2-dev \ libsasl2-modules \ libssl-dev \ libxmlsec1 \ libxmlsec1-dev \ locales \ openssh-client \ openssl \ pkg-config \ pkgconf \ sasl2-bin \ sqlite3 \ sudo \ tdsodbc \ unixodbc \ unixodbc-dev \ wget \ xz-utils \ zlib1g-dev \ " if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then # Needed only to compile Python from sources - without them it builds, but silently # lacks modules such as sqlite3, lzma or readline. local debian_version # shellcheck disable=SC1091 debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME") if [[ "${debian_version}" == "bookworm" ]]; then DEV_APT_DEPS+="lcov libbluetooth-dev libbz2-dev libdb-dev libgdbm-compat-dev libgdbm-dev \ liblzma-dev libncurses5-dev libreadline6-dev libsqlite3-dev libzstd-dev lzma lzma-dev tk-dev uuid-dev" else DEV_APT_DEPS+="lcov libbluetooth-dev libbz2-dev libdb-dev libgdbm-compat-dev libgdbm-dev \ liblzma-dev libncurses-dev libreadline-dev libsqlite3-dev libzstd-dev tk-dev uuid-dev" fi fi export DEV_APT_DEPS fi } function get_runtime_apt_deps() { local debian_version local debian_version_apt_deps # Get debian version without installing lsb_release # shellcheck disable=SC1091 debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) echo echo "DEBIAN CODENAME: ${debian_version}" echo if [[ "${debian_version}" == "bookworm" ]]; then debian_version_apt_deps="\ libffi8 \ libldap-2.5-0 \ libssl3 \ netcat-openbsd\ " else # trixie renamed the libraries that moved to a 64-bit time_t (libssl3 -> libssl3t64) and # dropped the soname from the LDAP library package name. debian_version_apt_deps="\ libffi8 \ libldap2 \ libssl3t64 \ netcat-openbsd\ " fi echo echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}" echo # libxmlsec1-openssl was added because libxmlsec1 ships no crypto engine of its own - the engines # are separate packages - so the "xmlsec" module (pulled in by python3-saml) imported with # "libxmlsec1-openssl.so.1: cannot open shared object file" without it. if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then RUNTIME_APT_DEPS="\ ${debian_version_apt_deps} \ apt-transport-https \ apt-utils \ curl \ dumb-init \ freetds-bin \ git \ gnupg \ iputils-ping \ krb5-user \ ldap-utils \ libev4 \ libgeos-dev \ libsasl2-2 \ libsasl2-modules \ libxmlsec1 \ libxmlsec1-openssl \ locales \ openssh-client \ rsync \ sasl2-bin \ sqlite3 \ sudo \ tdsodbc \ unixodbc \ wget\ " export RUNTIME_APT_DEPS fi } function install_docker_cli() { apt-get update apt-get install ca-certificates curl install -m 0755 -d /etc/apt/keyrings curl -fsSL --retry 3 --retry-delay 5 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc # shellcheck disable=SC1091 echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ tee /etc/apt/sources.list.d/docker.list > /dev/null apt-get update apt-get install -y --no-install-recommends docker-ce-cli } function keep_image_conffiles() { # The hardened base images ship a modified /etc/debian_version, so any apt run that upgrades # base-files stops at dpkg's interactive conffile prompt and fails a non-interactive build. Making # dpkg keep the image's copy by default covers every apt run - this build's and those of images # extending it - so no install command has to remember the flags. mkdir -p /etc/dpkg/dpkg.cfg.d printf '%s\n' force-confdef force-confold > /etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles } function prefer_hardened_apt_repository() { # The hardened base images take their packages from dhi.io, but stock Debian's sources stay # configured beside it and apt installs whichever repository offers the higher version. Debian's # build of a package then replaces Docker's, and its dependencies no longer match the packages the # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above the default priority of # 500 makes apt take Docker's build whenever dhi.io publishes the package, and fall back to Debian # only when it does not. # The priority stays below 1000 so the preference never forces a downgrade: apt still favors # dhi.io's build of a package it publishes, but may take a newer version from another repository # when a dependency requires it - for example PGDG's libpq5, which postgresql-client-18 needs at # ">= 18.6" and dhi.io does not carry. The renumbered builds that sort below the image's own copy # are handled by align_image_packages_with_hardened_repository instead. # The preference covers every apt run - this build's and those of images extending it. mkdir -p /etc/apt/preferences.d printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' > /etc/apt/preferences.d/airflow-prefer-dhi } function align_image_packages_with_hardened_repository() { # The dhi.io apt repository is live, so it moves on even under a base image pinned to a digest. # Docker renumbered its builds ("14.2.0-19+dhi5" became "14.2.0-19+deb13u0+dhi5", which sorts # lower) and the repository dropped the old numbers, while the image still ships them. Packages # depending on the exact new version (gcc-14 on gcc-14-base, libgmp-dev on libgmp10) are then # uninstallable, because apt will not downgrade the image's copy at a priority below 1000 # (docker-hardened-images/catalog#689). Only installed versions no repository offers any more # ("local" ones) are moved to dhi.io's build, so newer packages from other repositories - such as # PGDG's libpq5 - are left alone. local package local dhi_version local -a downgrades=() while read -r package; do dhi_version=$(apt-cache madison "${package}" | awk -F' [|] ' '$3 ~ /dhi\.io/ {print $2; exit}') if [[ -n "${dhi_version}" ]]; then downgrades+=("${package}=${dhi_version}") fi done < <(apt list --installed 2>/dev/null | awk -F/ '/\[installed,local\]/ {print $1}') if (( ${#downgrades[@]} )); then echo "Aligning image packages with the dhi.io repository: ${downgrades[*]}" apt-get install -y --no-install-recommends --allow-downgrades "${downgrades[@]}" fi } function refresh_ldconfig_after_dpkg() { # The hardened base images ship libc-bin without its dpkg trigger, so installing a shared library # does not refresh the ldconfig cache and ctypes.util.find_library() - which reads that cache - # cannot find it (for example pyenchant's "The 'enchant' C library was not found"). The hook covers # every apt run - this build's and those of images extending it. mkdir -p /etc/apt/apt.conf.d echo 'DPkg::Post-Invoke { "if [ -x /sbin/ldconfig ]; then /sbin/ldconfig; fi"; };' \ > /etc/apt/apt.conf.d/99airflow-ldconfig } function restore_debian_base_files() { # The hardened base images ship a minimal /etc, but Debian maintainer scripts assume the files # a stock Debian has: sasl2-bin chowns its run directory to the "sasl" group from base-passwd, # and tmux registers its shell with add-shell, which reads /etc/shells. The base-passwd package # only ships the reference copies of the account files - update-passwd is what merges them # into /etc. # libpam-runtime generates the /etc/pam.d/common-* files that the PAM configs already in the # image "@include" - without them "adduser --gecos" aborts with a PAM error from chfn. # Debian Policy lets a maintainer script rely on any essential package without declaring it, and # libgcrypt20 takes that up: its postinst runs a helper with a "#!/bin/dash" shebang. The hardened # images ship /bin/sh as a symlink to bash and no dash at all, so configuring the package aborts # with exit 127 and takes the whole apt transaction with it. Installing it first restores the # assumption, and hands /bin/sh back to dash the way a stock Debian has it. apt-get install -y --no-install-recommends dash apt-get install -y --no-install-recommends base-passwd libpam-runtime update-passwd # The Debian 13 hardened images strip /etc/pam.d further: libpam-runtime and passwd are already # installed there, so installing them generates nothing, and passwd, chpasswd and chfn abort with # "pam_start() failed". Reinstalling passwd brings back its deleted PAM configs, and # pam-auth-update - which treats the deleted common-* files as local changes - needs --force. apt-get install -y --no-install-recommends --reinstall -o Dpkg::Options::=--force-confmiss passwd if [[ ! -e /etc/pam.d/common-auth ]]; then pam-auth-update --package --force fi if [[ ! -e /etc/shells ]]; then printf '%s\n' "# /etc/shells: valid login shells" /bin/sh /bin/bash > /etc/shells fi align_perl_base_with_perl } function align_perl_base_with_perl() { # The hardened images ship their own build of perl-base, but the hardened apt repository does not # always publish the matching perl - while it still publishes packages that depend on perl, such # as git. Debian's perl depends on exactly its own perl-base version, so apt refuses to install # git at all. Moving perl-base to the version of the perl apt can install keeps the two in step; # when the repository does publish a matching perl, the versions already agree and this does nothing. local perl_version local perl_base_version perl_version=$(apt-cache policy perl | awk '/Candidate:/ {print $2}') perl_base_version=$(dpkg-query -W -f='${Version}' perl-base 2>/dev/null || true) if [[ -n "${perl_version}" && "${perl_version}" != "(none)" && -n "${perl_base_version}" \ && "${perl_base_version}" != "${perl_version}" ]]; then apt-get install -y --no-install-recommends --allow-downgrades "perl-base=${perl_version}" fi } function install_debian_dev_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1 if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then align_image_packages_with_hardened_repository restore_debian_base_files fi apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_DEV_APT_ENV?} if [[ ${DEV_APT_COMMAND} != "" ]]; then bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" fi if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" fi apt-get update local debian_version local debian_version_apt_deps # Get debian version without installing lsb_release # shellcheck disable=SC1091 debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";) echo echo "DEBIAN CODENAME: ${debian_version}" echo # shellcheck disable=SC2086 apt-get install -y --no-install-recommends ${DEV_APT_DEPS} } function install_additional_dev_dependencies() { if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then # shellcheck disable=SC2086 apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS} fi } function link_python() { # Airflow images have always exposed Python under /usr/python - documentation, volume mounts and # user customizations refer to that path - while the hardened base images ship it in /opt/python # (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink. if [[ ! -e /usr/python ]]; then ln -sv "${PYTHON_HOME}" /usr/python fi # The hardened base images have no /usr/local tree at all mkdir -p /usr/local/bin /usr/local/lib # link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix # Links in /usr/local/bin are needed for tools that expect python to be there # Links in /usr/python/bin are needed for tools that are detecting home of python installation including # lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part. for dst in pip3 python3 python3-config; do src="$(echo "${dst}" | tr -d 3)" if [[ ! -e "/usr/python/bin/${dst}" ]]; then continue fi echo "Linking ${dst} in /usr/local/bin and /usr/python/bin" ln -sfv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}" for dir in /usr/local/bin /usr/python/bin; do if [[ ! -e "${dir}/${src}" ]]; then echo "Creating ${src} - > ${dst} link in ${dir}" ln -sv "${dir}/${dst}" "${dir}/${src}" fi done done # A Python installed under /usr already has its libraries where the dynamic linker looks, and # linking all of /usr/lib into /usr/local/lib would only shadow the system libraries. if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then for dst in /usr/python/lib/* do src="/usr/local/lib/$(basename "${dst}")" if [[ -e "${src}" ]]; then rm -rf "${src}" fi echo "Linking ${dst} to ${src}" ln -sv "${dst}" "${src}" done fi ldconfig } function restore_thread_stack_size() { # The hardened base images build Python with -DTHREAD_STACK_SIZE=0x100000, so every thread Python # starts gets a 1 MiB stack instead of glibc's default (the 8 MiB "ulimit -s" the previous images # used). On Python 3.12 and 3.13 the C recursion guard is a fixed depth count sized for the larger # stack, so deeply nested input - e.g. a JSON body parsed in a web server worker thread - overflows # the stack and kills the process instead of raising RecursionError. A .pth file runs at every # interpreter start-up without taking the sitecustomize module name users may already rely on. local site_packages site_packages="$(/usr/python/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" echo "Restoring the 8 MiB default thread stack size in ${site_packages}" echo "import threading; threading.stack_size(8 * 1024 * 1024)" > "${site_packages}/airflow-thread-stack-size.pth" } function compile_python_stdlib() { # The hardened base images ship the standard library with no .pyc files at all. Python then misses # the bytecode cache on every stdlib import, and in a read-only or non-writable directory it cannot # create one - each miss leaves a negative dentry in the kernel, which grows without bound in a # long-running container and can exhaust memory on the host. Compiling the standard library here # restores what the previously compiled-in-image Python shipped. # See https://github.com/apache/airflow/pull/58944 and https://lwn.net/Articles/814535/ local stdlib stdlib="$(/usr/python/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["stdlib"])')" echo "Compiling Python standard library in ${stdlib}" # compileall exits non-zero when any file fails to compile, and the standard library ships files # that are meant not to compile (deliberately broken syntax used by the test suite). /usr/python/bin/python -m compileall -q -j "$(nproc)" -o 0 -o 1 -o 2 "${stdlib}" || true } function hold_hardened_python() { # The Debian 13 hardened images install their Python as packages that conflict with Debian's own # python3. A package depending on python3 - for example NodeSource's nodejs - therefore makes apt # remove the image's Python and install Debian's, without asking. Holding the packages turns that # into an apt error instead, in this build and in images extending it. local packages packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${Package}\n' 'python-3.*' 'libpython-3.*' 2>/dev/null \ | awk '$1 == "ii" {print $2}' || true) if [[ -n "${packages}" ]]; then # shellcheck disable=SC2086 apt-mark hold ${packages} fi } function check_no_system_python() { # Python - from the hardened base image or compiled from sources - must stay the only Python in the # image. A system Python pulled in as a dependency of an apt package shares its shared libraries # with ours and leads to errors such as: # /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add # Debian names its Python libraries "libpython3.13". The trixie hardened images register their own # Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match. # Debian 13 hardened images package their Python as "python-3.13" and serve other versions from # their apt repository, so a package depending on python3 can also pull in a hardened Python of a # different version than the image's own. local other_hardened_python="" if [[ -n "${BASE_PYTHON_MAJOR_MINOR=}" ]]; then other_hardened_python=$(dpkg -l | awk '/^ii (lib)?python-3\.[0-9]+/ {print $2}' \ | grep -v -E "python-${BASE_PYTHON_MAJOR_MINOR//./\\.}(-|:|$)" || true) fi if [[ -n "${other_hardened_python}" ]]; then echo echo "ERROR! A Python other than the image's ${BASE_PYTHON_MAJOR_MINOR} was installed: ${other_hardened_python//$'\n'/ }" echo apt-get install -yqq aptitude >/dev/null aptitude why "$(echo "${other_hardened_python}" | head -1 | cut -d: -f1)" echo exit 1 fi if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then echo echo "ERROR! System python is installed by one of the previous steps" echo echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:" echo apt-get install -yqq aptitude >/dev/null aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')" echo exit 1 else echo echo "GOOD! System python is not installed - OK" echo fi } function install_cosign() { local arch arch="$(dpkg --print-architecture)" declare -A cosign_sha256s=( # https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt [amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2" [arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab" ) local cosign_sha256="${cosign_sha256s[${arch}]}" if [[ -z "${cosign_sha256}" ]]; then echo "Unsupported architecture for cosign: ${arch}" exit 1 fi curl -fsSL --retry 3 --retry-delay 5 \ "https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" \ -o /tmp/cosign echo "${cosign_sha256} /tmp/cosign" | sha256sum --check chmod +x /tmp/cosign } function resolve_python_version() { if [[ -z "${AIRFLOW_PYTHON_VERSION}" ]]; then echo echo "ERROR! The legacy flavor compiles Python from sources, so AIRFLOW_PYTHON_VERSION has to name it:" echo "MAJOR.MINOR for the latest release of that line, or MAJOR.MINOR.PATCH." echo exit 1 fi if [[ ! "${AIRFLOW_PYTHON_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then return fi # A release directory appears on python.org with its first pre-release, so take the newest one # that already holds the final tarball. local candidate for candidate in $(wget --tries=3 --waitretry=5 -qO- https://www.python.org/ftp/python/ \ | grep -oE "href=\"${AIRFLOW_PYTHON_VERSION//./\\.}\.[0-9]+/\"" \ | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r); do if wget --tries=3 --waitretry=5 -q --spider \ "https://www.python.org/ftp/python/${candidate}/Python-${candidate}.tar.xz"; then echo "Python ${AIRFLOW_PYTHON_VERSION} resolved to its latest release ${candidate}" AIRFLOW_PYTHON_VERSION="${candidate}" return fi done echo echo "ERROR! No release of Python ${AIRFLOW_PYTHON_VERSION} found on python.org." echo exit 1 } function install_python() { # Only the legacy image flavor compiles Python - the hardened flavor gets it from the base image. resolve_python_version wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})" # Sigstore verification (PEP 761) declare -A sigstore_identities=( # https://peps.python.org/pep-0664/#release-manager-and-crew [3.11]="pablogsal@python.org" # https://peps.python.org/pep-0693/#release-manager-and-crew [3.12]="thomas@python.org" # https://peps.python.org/pep-0719/#release-manager-and-crew [3.13]="thomas@python.org" # https://peps.python.org/pep-0745/#release-manager-and-crew [3.14]="hugo@python.org" # https://peps.python.org/pep-0790/#release-manager-and-crew [3.15]="hugo@python.org" ) declare -A sigstore_issuers=( [3.11]="https://accounts.google.com" [3.12]="https://accounts.google.com" [3.13]="https://accounts.google.com" [3.14]="https://github.com/login/oauth" [3.15]="https://github.com/login/oauth" ) wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \ "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore" install_cosign local identity="${sigstore_identities[${major_minor_version}]}" local issuer="${sigstore_issuers[${major_minor_version}]}" /tmp/cosign verify-blob \ --bundle python.tar.xz.sigstore \ --certificate-identity "${identity}" \ --certificate-oidc-issuer "${issuer}" \ python.tar.xz rm -f python.tar.xz.sigstore /tmp/cosign mkdir -p /usr/src/python tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz rm python.tar.xz cd /usr/src/python arch="$(dpkg --print-architecture)"; arch="${arch##*-}" gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)" EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"; LDFLAGS="$(dpkg-buildflags --get LDFLAGS)" LDFLAGS="${LDFLAGS:--Wl},--strip-all" local build_log build_log=$(mktemp) echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..." if ! ( ./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \ --enable-loadable-sqlite-extensions --enable-option-checking=fatal \ --enable-shared --with-lto && \ make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \ "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \ make -s -j "$(nproc)" install ) > "${build_log}" 2>&1; then echo echo "ERROR! Python build failed. Build output:" echo cat "${build_log}" rm -f "${build_log}" exit 1 fi rm -f "${build_log}" # CPython builds without the optional modules whose -dev packages are missing, so check them here # rather than finding out from a broken image. /usr/python/bin/python3 -c "import bz2, ctypes, dbm.gnu, lzma, readline, sqlite3, ssl, uuid, zlib" cd / rm -rf /usr/src/python find /usr/python -depth \ \( \ \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ -o \( -type f -a \( -name 'libpython*.a' \) \) \ \) -exec rm -rf '{}' + link_python } function install_debian_runtime_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then align_image_packages_with_hardened_repository restore_debian_base_files fi apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates # shellcheck disable=SC2086 export ${ADDITIONAL_RUNTIME_APT_ENV?} if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" fi if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" fi apt-get update # shellcheck disable=SC2086 apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS} apt-get autoremove -yqq --purge apt-get clean check_no_system_python link_python if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then hold_hardened_python restore_thread_stack_size compile_python_stdlib fi rm -rf /var/lib/apt/lists/* /var/log/* } function install_golang() { curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz" rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz rm -f go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz } function install_jdk() { # Install Eclipse Temurin JDK from the Adoptium apt repository (https://adoptium.net/installation/linux/). apt-get update -qq apt-get install -y --no-install-recommends wget gnupg apt-transport-https ca-certificates mkdir -p /etc/apt/keyrings wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public \ | tee /etc/apt/keyrings/adoptium.asc > /dev/null # shellcheck disable=SC1091 DISTRO_CODENAME=$(. /etc/os-release; echo "${VERSION_CODENAME}") echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] \ https://packages.adoptium.net/artifactory/deb ${DISTRO_CODENAME} main" \ | tee /etc/apt/sources.list.d/adoptium.list > /dev/null apt-get update -qq apt-get install -y --no-install-recommends "temurin-${TEMURIN_VERSION}-jdk" apt-get clean rm -rf /var/lib/apt/lists/* } function install_nodejs() { local arch arch="$(dpkg --print-architecture)" declare -A nodejs_targets=( [amd64]="linux-x64" [arm64]="linux-arm64" ) declare -A nodejs_sha256s=( # https://nodejs.org/dist/v${NODEJS_VERSION}/SHASUMS256.txt [amd64]="9749e988f437343b7fa832c69ded82a312e41a03116d766797ac14f6f9eee578" [arm64]="0294e8b915ab75f92c7513d2fcb830ae06e10684e6c603e99a87dbf8835389c1" ) local target="${nodejs_targets[${arch}]}" local nodejs_sha256="${nodejs_sha256s[${arch}]}" if [[ -z "${target}" ]]; then echo "Unsupported architecture for nodejs: ${arch}" exit 1 fi curl --retry 3 --retry-delay 5 \ "https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-${target}.tar.xz" \ -o /tmp/nodejs.tar.xz echo "${nodejs_sha256} /tmp/nodejs.tar.xz" | sha256sum --check tar -xJf /tmp/nodejs.tar.xz --strip-components=1 -C /usr/local --no-same-owner rm -f /tmp/nodejs.tar.xz corepack enable --install-directory /usr/local/bin # corepack enable only writes shims; prepare downloads and caches the pnpm binary into the # image so it is available offline and matches the version ts-sdk pins. corepack prepare "pnpm@${PNPM_VERSION}" --activate } function install_rustup() { local arch arch="$(dpkg --print-architecture)" declare -A rustup_targets=( [amd64]="x86_64-unknown-linux-gnu" [arm64]="aarch64-unknown-linux-gnu" ) declare -A rustup_sha256s=( # https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256 [amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10" [arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792" ) local target="${rustup_targets[${arch}]}" local rustup_sha256="${rustup_sha256s[${arch}]}" if [[ -z "${target}" ]]; then echo "Unsupported architecture for rustup: ${arch}" exit 1 fi curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-delay 5 \ "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \ -o /tmp/rustup-init echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check chmod +x /tmp/rustup-init # Building wheels from source needs only rustc and cargo. The default profile also adds # rust-docs, clippy and rustfmt, which add tens of thousands of files to the image. /tmp/rustup-init -y --profile minimal --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}" rm -f /tmp/rustup-init } function apt_clean() { apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false rm -rf /var/lib/apt/lists/* /var/log/* } if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then get_runtime_apt_deps install_debian_runtime_dependencies install_docker_cli apt_clean else get_dev_apt_deps install_debian_dev_dependencies check_no_system_python if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then link_python hold_hardened_python restore_thread_stack_size compile_python_stdlib else install_python fi install_additional_dev_dependencies install_rustup if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then install_golang install_jdk install_nodejs fi install_docker_cli apt_clean fi EOF # The content below is automatically copied from scripts/docker/install_mysql.sh COPY <<"EOF" /install_mysql.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" set -euo pipefail common::get_colors declare -a packages : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}" if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then echo echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}" echo exit 1 fi if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then echo echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}" echo exit 1 fi if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then echo echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}" echo echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys," echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client." echo "" echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling" echo "with fixes and re-signing the packages which lasts few days" echo "See https://bugs.mysql.com/bug.php?id=113432 for more details." echo "As a community we are not able to support this broken packaging practice from Oracle" echo "Feel free however to install MySQL drivers on your own as extension of the image." echo exit 1 fi retry() { local retries=3 local count=0 # adding delay of 10 seconds local delay=10 until "$@"; do exit_code=$? count=$((count + 1)) if [[ $count -lt $retries ]]; then echo "Command failed. Attempt $count/$retries. Retrying in ${delay}s..." sleep $delay else echo "Command failed after $retries attempts." return $exit_code fi done } install_mariadb_client() { # https://mariadb.org/about/#maintenance-policy # MariaDB publishes each LTS series only for the Debian releases current when it came out - 10.11 # has no trixie repository and 11.8 is the first LTS that does. local mariadb_lts_version="10.11" if [[ "$(common::debian_codename)" != "bookworm" ]]; then mariadb_lts_version="11.8" fi # List of compatible package Oracle MySQL -> MariaDB: # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+) # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat` # `libmysqlclient-dev` -> `libmariadb-dev-compat` # # Different naming against Debian repo which we used before # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo: # `libmariadb-dev` -> `libmariadb-dev-compat` # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+) if [[ "${1}" == "dev" ]]; then packages=("libmariadb-dev-compat" "mariadb-client") elif [[ "${1}" == "prod" ]]; then packages=("libmariadb3-compat" "mariadb-client") else echo echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi if [[ "${mariadb_lts_version%%.*}" -ge 11 ]]; then # From 11.0 mariadb-client no longer ships the "mysql" named commands packages+=("mariadb-client-compat") fi common::import_trusted_gpg "0xF1656F24C74CD1D8" "mariadb" echo echo "${COLOR_BLUE}Installing MariaDB client version ${mariadb_lts_version}: ${1}${COLOR_RESET}" echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${mariadb_lts_version}/repo/debian/ $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mariadb.list # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb retry apt-get update retry apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* } if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then install_mariadb_client "${@}" fi EOF # The content below is automatically copied from scripts/docker/install_mssql.sh COPY <<"EOF" /install_mssql.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" set -euo pipefail common::get_colors declare -a packages : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" function install_mssql_client() { # Install MsSQL client from Microsoft repositories if [[ ${INSTALL_MSSQL_CLIENT:="true"} != "true" ]]; then echo echo "${COLOR_BLUE}Skip installing mssql client${COLOR_RESET}" echo return fi packages=("msodbcsql18") common::import_trusted_gpg "EB3E94ADBE1229CF" "microsoft" # The Debian 13 repository is signed with Microsoft's newer key, the older releases with the one above. common::import_trusted_gpg "EE4D7792F748182B" "microsoft-2025" echo echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}" echo echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(common::debian_release)/prod $(common::debian_codename) main" > \ /etc/apt/sources.list.d/mssql-release.list && mkdir -p /opt/microsoft/msodbcsql18 && touch /opt/microsoft/msodbcsql18/ACCEPT_EULA && apt-get update -yqq && apt-get upgrade -yqq && apt-get -yqq install --no-install-recommends "${packages[@]}" && apt-get autoremove -yqq --purge && apt-get clean && rm -rf /var/lib/apt/lists/* } install_mssql_client "${@}" EOF # The content below is automatically copied from scripts/docker/install_postgres.sh COPY <<"EOF" /install_postgres.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" set -euo pipefail common::get_colors declare -a packages : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" install_postgres_client() { echo echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}" echo if [[ "${1}" == "dev" ]]; then packages=("libpq-dev" "postgresql-client") elif [[ "${1}" == "prod" ]]; then packages=("postgresql-client") else echo echo "Specify either prod or dev" echo exit 1 fi common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres" echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list # The hardened image's airflow-prefer-dhi pin outranks Debian's default, so apt would otherwise keep # dhi.io's libpq (e.g. 17.11) and postgresql-client (which needs a matching, newer libpq5) could not # install. PostgreSQL's own repository is the authority for its packages - rank it above the dhi.io # preference so apt takes libpq from PGDG. mkdir -p /etc/apt/preferences.d printf 'Package: *\nPin: origin "apt.postgresql.org"\nPin-Priority: 1001\n' > /etc/apt/preferences.d/airflow-prefer-pgdg apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge apt-get clean && rm -rf /var/lib/apt/lists/* } if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then install_postgres_client "${@}" fi EOF # The content below is automatically copied from scripts/docker/install_packaging_tools.sh COPY <<"EOF" /install_packaging_tools.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" common::get_colors common::get_packaging_tool common::show_packaging_tool_version_and_location common::install_packaging_tools EOF # The content below is automatically copied from scripts/docker/common.sh COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' COLOR_RED=$'\e[31m' COLOR_RESET=$'\e[0m' COLOR_YELLOW=$'\e[33m' export COLOR_BLUE export COLOR_GREEN export COLOR_RED export COLOR_RESET export COLOR_YELLOW } function common::get_packaging_tool() { : "${AIRFLOW_USE_UV:?Should be set}" ## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN ## `scripts/in_container/_in_container_utils.sh` if [[ ${AIRFLOW_USE_UV} == "true" ]]; then echo echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 # (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then # for uv only install dev group when we install from sources export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec" else export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec" fi export EXTRA_UNINSTALL_FLAGS="" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest" export UPGRADE_IF_NEEDED="--upgrade" UV_CONCURRENT_DOWNLOADS=$(nproc --all) export UV_CONCURRENT_DOWNLOADS if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary" fi else echo echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" echo export PACKAGING_TOOL="pip" export PACKAGING_TOOL_CMD="pip" # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 # (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 export EXTRA_INSTALL_FLAGS="--root-user-action ignore --no-binary lxml,xmlsec" export EXTRA_UNINSTALL_FLAGS="--yes" export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy eager" export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed" if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre" fi fi } function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} && -n ${AIRFLOW_VERSION} && ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}" fi } function common::get_constraints_location() { # When installing from sources without upgrade, generate constraints from uv.lock if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -z "${UPGRADE_RANDOM_INDICATOR_STRING=}" ]]; then echo echo "${COLOR_BLUE}Installing from sources with uv.lock - generating constraints from uv.lock${COLOR_RESET}" echo uv export --frozen --no-hashes --no-emit-project --no-editable --no-header \ --no-annotate > "${HOME}/constraints.txt" 2>/dev/null || true return fi # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} else AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH} fi fi if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}" local python_version python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then echo echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" echo if ! curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}"; then echo echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION} (new Python version being bootstrapped?).${COLOR_RESET}" echo "${COLOR_YELLOW}Falling back to no-constraints installation.${COLOR_RESET}" echo AIRFLOW_CONSTRAINTS_LOCATION="" # Create an empty constraints file so --constraint flag still works touch "${HOME}/constraints.txt" fi else echo echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}" echo cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt" fi } function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" echo "Installed pip: $(pip --version): $(command -v pip)" if [[ ${PACKAGING_TOOL} == "pip" ]]; then echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" else echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed yet"): $(command -v uv 2>/dev/null)" fi } function common::install_packaging_tools() { : "${AIRFLOW_USE_UV:?Should be set}" if [[ "${VIRTUAL_ENV=}" != "" ]]; then echo echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}" echo else echo echo "${COLOR_BLUE}Checking packaging tools for system Python installation: $(command -v python)${COLOR_RESET}" echo fi if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then echo echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}" echo pip install --root-user-action ignore --disable-pip-version-check --upgrade pip elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then echo echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else local installed_pip_version installed_pip_version=$(python -c 'from importlib.metadata import version; print(version("pip"))') if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then echo echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" fi fi if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then echo echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}" echo pip install --root-user-action ignore --disable-pip-version-check --upgrade uv elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then echo echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}" echo # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" else local installed_uv_version installed_uv_version=$(python -c 'from importlib.metadata import version; print(version("uv"))' 2>/dev/null || echo "Not installed yet") if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then echo echo "${COLOR_BLUE}(Re)Installing uv version: ${AIRFLOW_UV_VERSION}${COLOR_RESET}" echo # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi fi if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then echo echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}" echo uv tool install prek --with uv # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" else echo echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}" echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}" echo uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}" # make sure that the venv/user in .local exists mkdir -p "${HOME}/.local/bin" fi } function common::debian_codename() { # shellcheck disable=SC1091 (. /etc/os-release; printf '%s\n' "${VERSION_CODENAME}") } function common::debian_release() { # shellcheck disable=SC1091 (. /etc/os-release; printf '%s\n' "${VERSION_ID}") } function common::import_trusted_gpg() { common::get_colors local key=${1:?${COLOR_RED}First argument expects OpenPGP Key ID${COLOR_RESET}} local name=${2:?${COLOR_RED}Second argument expected trust storage name${COLOR_RESET}} local key_file="/scripts/docker/keys/${name}.asc" echo "${COLOR_BLUE}Installing GPG public key ${key} from ${key_file}${COLOR_RESET}" gpg --dearmor < "${key_file}" > "/etc/apt/trusted.gpg.d/${name}.gpg" } EOF # The content below is automatically copied from scripts/docker/install_airflow_when_building_images.sh COPY <<"EOF" /install_airflow_when_building_images.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" function install_from_sources() { local extra_sync_flags extra_sync_flags="" if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then echo echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" echo return fi extra_sync_flags="${extra_sync_flags} --no-install-workspace" fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x echo echo "${COLOR_RED}We only support uv not pip installation for upgrading dependencies!.${COLOR_RESET}" echo exit 1 fi set +x echo echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}" echo # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 # (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 set -x uv sync --all-packages --resolution highest --group ci-image \ ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 # shellcheck disable=SC2206 # extra_sync_flags holds several flags local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec --no-python-downloads --no-managed-python) set +x # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; # the installation that adds the workspace packages does that later. if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then echo echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" echo return fi echo echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" echo set -x if ! "${frozen_sync[@]}"; then set +x if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then echo echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" echo exit 1 fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" echo exit 1 fi echo echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies not reflected in uv.lock.${COLOR_RESET}" echo echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}" echo set -x uv sync --all-packages --group ci-image \ ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python set +x fi fi } function install_from_external_spec() { local installation_command_flags if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" else echo echo "${COLOR_RED}The '${AIRFLOW_INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}" echo echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow')${COLOR_RESET}" echo exit 1 fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then echo echo "${COLOR_BLUE}Remove airflow and all provider distributions installed before potentially${COLOR_RESET}" echo set -x ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} set +x else echo echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}" echo set -x if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then set +x if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}" echo exit 1 fi echo echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies conflicting with constraints.${COLOR_RESET}" echo echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}" echo set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} set +x fi fi } function install_airflow_when_building_images() { # Remove mysql from extras if client is not going to be installed if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} echo "${COLOR_YELLOW}MYSQL client installation is disabled. Extra 'mysql' installations were therefore omitted.${COLOR_RESET}" fi # Remove postgres from extras if client is not going to be installed if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,} echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra 'postgres' installations were therefore omitted.${COLOR_RESET}" fi # Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then install_from_sources else install_from_external_spec fi set +x if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages return fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo # We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip` pip check } common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location common::show_packaging_tool_version_and_location install_airflow_when_building_images EOF # The content below is automatically copied from scripts/docker/install_additional_dependencies.sh COPY <<"EOF" /install_additional_dependencies.sh #!/usr/bin/env bash set -euo pipefail : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" function install_additional_dependencies() { if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then echo echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo # We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip` pip check else echo echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} set +x common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo # We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip` pip check fi } common::get_colors common::get_packaging_tool common::get_airflow_version_specification common::get_constraints_location common::show_packaging_tool_version_and_location install_additional_dependencies EOF # The content below is automatically copied from scripts/docker/entrypoint_ci.sh COPY <<"EOF" /entrypoint_ci.sh #!/usr/bin/env bash function set_verbose() { if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then set -x else set +x fi } set_verbose . "${AIRFLOW_SOURCES:-/opt/airflow}"/scripts/in_container/_in_container_script_init.sh LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6" export LD_PRELOAD chmod 1777 /tmp AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd) PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.11} export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} mkdir "${AIRFLOW_HOME}/sqlite" -p || true ASSET_COMPILATION_WAIT_MULTIPLIER=${ASSET_COMPILATION_WAIT_MULTIPLIER:=1} if [[ "${CI=}" == "true" ]]; then export COLUMNS="202" fi . "${IN_CONTAINER_DIR}/check_connectivity.sh" function wait_for_asset_compilation() { if [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; then echo echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}" echo local counter=0 while [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; do if (( counter % 5 == 2 )); then echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}" fi sleep 1 ((counter=counter+1)) if [[ ${counter} == 30*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then echo echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}" echo """ If it does not complete soon, you might want to stop it and remove file lock: * press Ctrl-C * run 'rm ${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock' """ fi if [[ ${counter} == 60*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then echo echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}" echo "${COLOR_RED}refer to dev/breeze/doc/04_troubleshooting.rst for resolution steps.${COLOR_RED}" echo exit 1 fi done fi if [ -f "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out" ]; then echo echo "${COLOR_RED}The asset compilation failed. Exiting.${COLOR_RESET}" echo cat "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out" rm "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out" echo exit 1 fi } function environment_initialization() { if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then return fi echo echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}" echo echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}" echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}" echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}" if [[ ${BACKEND=} == "postgres" ]]; then echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Postgres: ${POSTGRES_VERSION}" elif [[ ${BACKEND=} == "mysql" ]]; then echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}" elif [[ ${BACKEND=} == "sqlite" ]]; then echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite" elif [[ ${BACKEND=} == "custom" ]]; then local _conn_url="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-}" local _masked_url _masked_url=$(echo "${_conn_url}" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|') echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Custom (${_masked_url})" fi echo if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then echo echo "${COLOR_BLUE}Forcing scheduler/standalone_dag_processor to True${COLOR_RESET}" echo export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True fi RUN_TESTS=${RUN_TESTS:="false"} CI=${CI:="false"} # Added to have run-tests on path export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/ mkdir -pv "${AIRFLOW_HOME}/logs/" # Change the default worker_concurrency for tests export AIRFLOW__CELERY__WORKER_CONCURRENCY=8 set +e # shellcheck source=scripts/in_container/configure_environment.sh . "${IN_CONTAINER_DIR}/configure_environment.sh" # shellcheck source=scripts/in_container/run_init_script.sh . "${IN_CONTAINER_DIR}/run_init_script.sh" "${IN_CONTAINER_DIR}/check_environment.sh" ENVIRONMENT_EXIT_CODE=$? set -e if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then echo echo "Error: check_environment returned ${ENVIRONMENT_EXIT_CODE}. Exiting." echo exit ${ENVIRONMENT_EXIT_CODE} fi mkdir -p /usr/lib/google-cloud-sdk/bin touch /usr/lib/google-cloud-sdk/bin/gcloud ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then # Set up the ssh-to-localhost plumbing used by the SSH/SFTP provider tests under a # dedicated directory instead of ~/.ssh: `breeze --forward-credentials` bind-mounts # the host's ~/.ssh at /root/.ssh and container startup must never modify it. breeze_ssh_dir="/root/.breeze-ssh" mkdir -p "${breeze_ssh_dir}" chmod 700 "${breeze_ssh_dir}" if [[ ! -f "${breeze_ssh_dir}/id_rsa" ]]; then ssh-keygen -t rsa -C airflow-breeze-internal-key -m PEM -P '' -f "${breeze_ssh_dir}/id_rsa" \ >"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1 fi cp "${breeze_ssh_dir}/id_rsa.pub" "${breeze_ssh_dir}/authorized_keys" chmod 600 "${breeze_ssh_dir}/id_rsa" "${breeze_ssh_dir}/authorized_keys" echo "AuthorizedKeysFile ${breeze_ssh_dir}/authorized_keys .ssh/authorized_keys" \ > /etc/ssh/sshd_config.d/airflow-breeze.conf # The heredoc delimiter must not be "EOF": Dockerfile.ci inlines this script inside # a COPY <<"EOF" heredoc and a bare EOF line would terminate it early. cat > /etc/ssh/ssh_config.d/airflow-breeze.conf </dev/null ssh-add "${breeze_ssh_dir}/id_rsa" >>"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1 sudo service ssh restart >/dev/null 2>&1 # Sometimes the server is not quick enough to load the keys! OpenSSH 10 also prints a # "# localhost:22 SSH-2.0-..." comment line per connection to stdout, so count only the keys. while [[ $(ssh-keyscan -H localhost 2>/dev/null | grep -cv '^#') != "3" ]] ; do echo "Not all keys yet loaded by the server" sleep 0.05 done ssh-keyscan -H localhost > "${breeze_ssh_dir}/known_hosts" 2>/dev/null fi if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then echo echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}" echo # Define LocalStack AWS configuration declare -A localstack_config=( ["AWS_ENDPOINT_URL"]="http://localstack:4566" ["AWS_ACCESS_KEY_ID"]="test" ["AWS_SECRET_ACCESS_KEY"]="test" ["AWS_DEFAULT_REGION"]="us-east-1" ) # Export each configuration variable and log it for key in "${!localstack_config[@]}"; do export "$key"="${localstack_config[$key]}" echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}" done echo fi cd "${AIRFLOW_SOURCES}" # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags # in the UI when testing in Breeze. This might be solved differently in the future if [[ -d /opt/airflow/providers/standard/tests ]]; then export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests fi if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then if [[ ${BREEZE_DEBUG_CELERY_WORKER=} == "true" ]]; then export AIRFLOW__CELERY__POOL=${AIRFLOW__CELERY__POOL:-solo} fi export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} if [[ ${SKIP_ASSETS_COMPILATION:="false"} == "false" ]]; then wait_for_asset_compilation fi if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then # shellcheck source=scripts/in_container/bin/run_mprocs exec run_mprocs else # shellcheck source=scripts/in_container/bin/run_tmux exec run_tmux fi fi } function handle_mount_sources() { if [[ ${MOUNT_SOURCES=} == "remove" ]]; then echo echo "${COLOR_BLUE}Mounted sources are removed, cleaning up mounted dist-info files${COLOR_RESET}" echo rm -rf /usr/local/lib/python"${PYTHON_MAJOR_MINOR_VERSION}"/site-packages/apache_airflow*.dist-info/ fi } function determine_airflow_to_use() { USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}" if [[ "${USE_AIRFLOW_VERSION}" == "" && "${USE_DISTRIBUTIONS_FROM_DIST=}" != "true" ]]; then export PYTHONPATH=${AIRFLOW_SOURCES} echo echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo # Cleanup the logs, tmp when entering the environment sudo rm -rf "${AIRFLOW_SOURCES}"/logs/* sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/* mkdir -p "${AIRFLOW_SOURCES}"/logs/ mkdir -p "${AIRFLOW_SOURCES}"/tmp/ else if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then echo echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}" echo # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | grep -v "@" | \ xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} # Now install rich ad click first to use the installation script # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} rich rich-click click \ --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt fi echo echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}" echo # Generate constraints from uv.lock and use them to install development dependencies # via the Python script. --no-cache is needed - otherwise there is possibility of # overriding temporary environments by multiple parallel processes # --all-packages: without it the export only covers the root project's dependencies. # Anything that only a workspace member's dependency group pulls in, such # as mcp through common.ai's pydantic-ai-slim[mcp], stays unpinned and resolves fresh # from PyPI instead of following uv.lock. local constraint_file="/tmp/constraints-from-lock.txt" uv export --frozen --no-hashes --no-emit-project --no-emit-workspace --no-editable --no-header \ --no-annotate --all-packages > "${constraint_file}" 2>/dev/null || true uv run --no-cache /opt/airflow/scripts/in_container/install_development_dependencies.py \ --constraint "${constraint_file}" # Some packages might leave legacy typing module which causes test issues # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true echo echo "${COLOR_BLUE}Installing airflow and providers ${COLOR_RESET}" echo python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py" fi if [[ "${USE_AIRFLOW_VERSION}" =~ ^2.* ]]; then # Remove auth manager setting unset AIRFLOW__CORE__AUTH_MANAGER fi if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then # make sure old variable is used for older airflow versions export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}" fi } function check_boto_upgrade() { if [[ ${UPGRADE_BOTO=} != "true" ]]; then return fi echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true # Urllib 2.6.0 broke kubernetes client by removing getheaders() (restored in 2.6.1+). # Exclude exactly 2.6.0; any other version is fine. # Tracked in https://github.com/kubernetes-client/python/issues/2477 # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "boto3" "botocore" "urllib3!=2.6.0" } function check_upgrade_sqlalchemy() { if [[ "${UPGRADE_SQLALCHEMY=}" != "true" ]]; then return fi echo echo "${COLOR_BLUE}Upgrading sqlalchemy to the latest version to run tests with it${COLOR_RESET}" echo uv sync --all-packages --resolution highest --no-python-downloads --no-managed-python } function check_downgrade_sqlalchemy() { if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then return fi local min_sqlalchemy_version min_sqlalchemy_version=$(grep "sqlalchemy\[asyncio\]>=" airflow-core/pyproject.toml | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs) echo echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" echo # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy[asyncio]==${min_sqlalchemy_version}" echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo # We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip` pip check } function check_downgrade_pendulum() { if [[ ${DOWNGRADE_PENDULUM=} != "true" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.12" ]]; then return fi local min_pendulum_version min_pendulum_version=$(grep "pendulum>=" airflow-core/pyproject.toml | head -1 | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs) echo echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}" echo # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}" echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo # We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip` pip check } function check_run_tests() { if [[ ${RUN_TESTS=} != "true" ]]; then return fi if [[ ${TEST_GROUP:=""} == "system" ]]; then exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}" else exec "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${@}" fi } function reinstall_shared_distributions() { # The shared distributions under shared// are workspace members that are not # transitively required by airflow-core or any provider, so the lowest-direct # `uv sync` above wipes them out from the environment. Re-install them with # --no-deps so that the `airflow_shared.*` namespace used by devel-common test # utils resolves at test collection time without disturbing the lowest-direct # resolution that was just applied. echo echo "${COLOR_BLUE}Re-installing shared distributions (airflow_shared.*) after uv sync${COLOR_RESET}" echo # shellcheck disable=SC2046 uv pip install --no-deps $(ls -d /opt/airflow/shared/*/) } PROVIDERS_NEEDING_PRE_EXTRAS_INSTALL=() function run_pre_extras_install_if_registered() { local provider_id="${1}" local registered_provider for registered_provider in "${PROVIDERS_NEEDING_PRE_EXTRAS_INSTALL[@]}"; do if [[ "${registered_provider}" == "${provider_id}" ]]; then echo echo "${COLOR_BLUE}Running pre-extras install manifest for ${provider_id}${COLOR_RESET}" echo local env_file env_file=$(mktemp) if ! python "${AIRFLOW_SOURCES}/scripts/in_container/run_pre_extras_install.py" \ "${provider_id}" --emit-env-to "${env_file}"; then rm -f "${env_file}" echo "${COLOR_RED}Pre-extras install failed for ${provider_id}${COLOR_RESET}" exit 1 fi if [[ -s "${env_file}" ]]; then # shellcheck disable=SC1090 source "${env_file}" fi rm -f "${env_file}" return fi done } function check_force_lowest_dependencies() { if [[ ${FORCE_LOWEST_DEPENDENCIES=} != "true" ]]; then return fi if [[ ${TEST_TYPE=} =~ Providers\[.*\] ]]; then local provider_id # shellcheck disable=SC2001 provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/') # A split provider's test type names one part of its tests: Providers[:] provider_id=${provider_id%%:*} echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}" echo if ! /opt/airflow/scripts/in_container/is_provider_excluded.py "${provider_id}"; then echo echo "S${COLOR_YELLOW}Skipping ${provider_id} provider check on Python ${PYTHON_MAJOR_MINOR_VERSION}!${COLOR_RESET}" echo exit 0 fi cd "${AIRFLOW_SOURCES}/providers/${provider_id/.//}" || exit 1 run_pre_extras_install_if_registered "${provider_id}" # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 # (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 local sync_successful="false" for attempt in 1 2 3; do echo "Attempt ${attempt} of syncing to lowest dependencies" set -x if UV_LOCK_TIMEOUT=200 uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \ --no-python-downloads --no-managed-python; then set +x sync_successful="true" break fi set +x echo "Sleeping 30s" sleep 30 echo "Attempt ${attempt} failed. Retrying..." done if [[ "${sync_successful}" != "true" ]]; then echo "${COLOR_RED}Failed to sync lowest dependencies after 3 attempts.${COLOR_RESET}" exit 1 fi else echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for Airflow.${COLOR_RESET}" echo cd "${AIRFLOW_SOURCES}/airflow-core" # --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2 # (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \ --no-python-downloads --no-managed-python fi reinstall_shared_distributions } function check_airflow_python_client_installation() { if [[ ${INSTALL_AIRFLOW_PYTHON_CLIENT=} != "true" ]]; then return fi python "${IN_CONTAINER_DIR}/install_airflow_python_client.py" } function initialize_db() { # If we are going to start the api server OR we are a system test (which may or may not start the api server, # depending on the Airflow version being used to run the tests), then migrate the DB. if [[ ${START_API_SERVER_WITH_EXAMPLES=} == "true" || ${TEST_GROUP:=""} == "system" ]]; then echo echo "${COLOR_BLUE}Initializing database${COLOR_RESET}" echo airflow db migrate echo echo "${COLOR_BLUE}Database initialized${COLOR_RESET}" fi } function start_api_server_with_examples(){ USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}" # Do not start the api server if either START_API_SERVER_WITH_EXAMPLES is false or the TEST_GROUP env var is not # equal to "system". if [[ ${START_API_SERVER_WITH_EXAMPLES=} != "true" && ${TEST_GROUP:=""} != "system" ]]; then return fi # If the use Airflow version is set and it is <= 3.0.0 (which does not have the API server anyway) also return if [[ ${USE_AIRFLOW_VERSION} != "" && ${USE_AIRFLOW_VERSION} < "3.0.0" ]]; then return fi export AIRFLOW__CORE__LOAD_EXAMPLES=True export AIRFLOW__API__EXPOSE_CONFIG=True airflow dags reserialize echo "Example dags parsing finished" if airflow config get-value core auth_manager | grep -q "FabAuthManager"; then echo "Create admin user" airflow users create -u admin -p admin -f Thor -l Administrator -r Admin -e admin@email.domain || true echo "Admin user created" else echo "Skipping user creation as auth manager different from Fab is used" fi echo echo "${COLOR_BLUE}Starting airflow api server${COLOR_RESET}" echo if [[ ${START_API_SERVER_DAEMON:-"true"} == "false" ]]; then airflow api-server --port 8080 & else airflow api-server --port 8080 --daemon fi echo echo "${COLOR_BLUE}Waiting for api-server to start${COLOR_RESET}" echo check_service_connection "Airflow api-server" "run_nc localhost 8080" 100 EXIT_CODE=$? if [[ ${EXIT_CODE} != 0 ]]; then echo echo "${COLOR_RED}Api server did not start properly${COLOR_RESET}" echo exit ${EXIT_CODE} fi echo echo "${COLOR_BLUE}Airflow api-server started${COLOR_RESET}" } handle_mount_sources determine_airflow_to_use environment_initialization check_boto_upgrade check_upgrade_sqlalchemy check_downgrade_sqlalchemy check_downgrade_pendulum check_force_lowest_dependencies check_airflow_python_client_installation initialize_db start_api_server_with_examples check_run_tests "${@}" exec /bin/bash "${@}" EOF # The content below is automatically copied from scripts/docker/entrypoint_exec.sh COPY <<"EOF" /entrypoint_exec.sh #!/usr/bin/env bash . /opt/airflow/scripts/in_container/_in_container_script_init.sh . /opt/airflow/scripts/in_container/configure_environment.sh . /opt/airflow/scripts/in_container/run_init_script.sh if [[ -S /root/.breeze-ssh/agent.sock ]]; then export SSH_AUTH_SOCK="/root/.breeze-ssh/agent.sock" fi exec /bin/bash "${@}" EOF # Extract manifests separately so COPY below depends on their contents, not all source files. FROM ${BASE_IMAGE} as dependency-manifests WORKDIR /source SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] RUN --mount=type=bind,target=/source \ manifest_list="$(mktemp)" && \ trap 'rm -f "${manifest_list}"' EXIT && \ find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \ mkdir -p /dependency-manifests && \ while IFS= read -r -d '' manifest; do \ relative_path="${manifest#./}"; \ mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \ cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \ done < "${manifest_list}" && \ if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi FROM ${BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example # xtrace - to show commands executed) SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ARG BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies. # NOTE! When you want to make sure dependencies are installed from scratch in your PR after removing # some dependencies, you also need to set "disable image cache" in your PR to make sure the image is # not built using the "main" version of those dependencies. ARG DEPENDENCIES_EPOCH_NUMBER="15" # Make sure noninteractive debian install is used and language variables set ENV BASE_IMAGE=${BASE_IMAGE} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \ DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \ INSTALL_MYSQL_CLIENT="true" \ INSTALL_MSSQL_CLIENT="true" \ INSTALL_POSTGRES_CLIENT="true" \ PIP_CACHE_DIR=/root/.cache/pip \ UV_CACHE_DIR=/root/.cache/uv # GCC 14 in trixie turns incompatible-pointer-types, int-conversion and implicit function declarations # into errors, and the hardened Python passes -Werror=implicit-function-declaration on to C extensions. # The lowest-dependency tests build old sdists (for example xmlsec 1.3.14 or ibm-db 3.2.0) that only # compiled because those were warnings - extensions append CFLAGS after Python's own flags, so this # turns them back into warnings for every build in the CI image. ENV CFLAGS="-Wno-error=incompatible-pointer-types -Wno-error=int-conversion -Wno-error=implicit-function-declaration" RUN echo "Base image version: ${BASE_IMAGE}" ARG DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_ENV_VARS="" # No libenchant-2-dev: pyenchant finds libenchant-2-2 through the ldconfig cache, and on trixie the # -dev package's build dependencies pull in a second Python from the hardened images' repository. ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git gdb graphviz krb5-user \ less libenchant-2-2 libgeos-dev libpq-dev net-tools \ netcat-openbsd openssh-server postgresql-client rsync tmux \ unzip vim xxd" ARG ADDITIONAL_DEV_APT_ENV="" ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} # Empty takes the Python of BASE_IMAGE, breeze passes the patchlevel of the image it builds on. ARG AIRFLOW_PYTHON_VERSION="" ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION} ENV GOLANG_MAJOR_MINOR_VERSION="1.27.3" ENV TEMURIN_VERSION="11" ENV NODEJS_VERSION="22.23.1" # keep this in sync with package.json files ENV PNPM_VERSION="10.28.1" ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/usr/local/cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh ci # In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python # libraries will be accidentally used before the library installed from sources (which is newer and # python interpreter might break if accidentally the old system libraries are used. ENV LD_LIBRARY_PATH="/usr/python/lib" COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other # scripts which are needed much later will not invalidate the docker layer here. COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/ COPY scripts/docker/keys/ /scripts/docker/keys/ ARG HOME=/root ARG AIRFLOW_IMAGE_TYPE="ci" ARG AIRFLOW_HOME=/root/airflow ARG AIRFLOW_SOURCES=/opt/airflow ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ENV HOME=${HOME} \ AIRFLOW_IMAGE_TYPE=${AIRFLOW_IMAGE_TYPE} \ AIRFLOW_HOME=${AIRFLOW_HOME} \ AIRFLOW_SOURCES=${AIRFLOW_SOURCES} \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} # We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an # unexpected result - the cache for Dockerfiles might get invalidated in case the host system # had different umask set and group x bit was not set. In Azure the bit might be not set at all. # That also protects against AUFS Docker backend problem where changing the executable bit required sync RUN bash /scripts/docker/install_mysql.sh prod \ && bash /scripts/docker/install_mysql.sh dev \ && bash /scripts/docker/install_mssql.sh dev \ && bash /scripts/docker/install_postgres.sh dev \ # The user is added to allow ssh debugging (you can connect with airflow/airflow over ssh) && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ --quiet "airflow" --home "/home/airflow" \ && echo -e "airflow\nairflow" | passwd airflow 2>&1 \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow # Install Helm ARG HELM_VERSION="v3.19.0" RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ && PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" ) \ && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-${PLATFORM}.tar.gz" \ && curl --silent --location --fail --retry 3 --retry-delay 5 -o /tmp/helm.tar.gz "${HELM_URL}" \ && tar -xzf /tmp/helm.tar.gz -O "${SYSTEM}-${PLATFORM}/helm" > /usr/local/bin/helm \ && rm -f /tmp/helm.tar.gz \ && chmod +x /usr/local/bin/helm # Install mprocs - a modern process manager for managing multiple Airflow components ARG MPROCS_VERSION="0.10.3" RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ && PLATFORM="$(uname -m)" \ && MPROCS_URL="https://github.com/pvolok/dekit/releases/download/v${MPROCS_VERSION}/dekit-${PLATFORM}-unknown-${SYSTEM}-musl.tar.gz" \ && echo "Downloading mprocs from ${MPROCS_URL}" \ && curl --silent --location --fail --retry 3 --retry-delay 5 -o /tmp/mprocs.tar.gz "${MPROCS_URL}" \ && tar -xzf /tmp/mprocs.tar.gz -O dekit > /usr/local/bin/mprocs \ && rm -f /tmp/mprocs.tar.gz \ && chmod +x /usr/local/bin/mprocs WORKDIR ${AIRFLOW_SOURCES} RUN mkdir -pv ${AIRFLOW_HOME} && \ mkdir -pv ${AIRFLOW_HOME}/dags && \ mkdir -pv ${AIRFLOW_HOME}/logs ARG AIRFLOW_REPO=apache/airflow ARG AIRFLOW_BRANCH=main # Airflow Extras installed ARG AIRFLOW_EXTRAS="all" ARG ADDITIONAL_AIRFLOW_EXTRAS="" # Allows to override constraints source ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" # By default fallback to installation without constraints because in CI image it should always be tried ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true" # By changing the epoch we can force reinstalling Airflow and pip all dependencies # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable. ARG AIRFLOW_CI_BUILD_EPOCH="10" # Setup PIP # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" # Optimizing installation of Cassandra driver (in case there are no prebuilt wheels which is the # case as of 20.04.2021 with Python 3.9 # Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes ARG CASS_DRIVER_NO_CYTHON="1" # Build cassandra driver on multiple CPUs ARG CASS_DRIVER_BUILD_CONCURRENCY="8" # This value should be set by the CI image build system to the current timestamp ARG AIRFLOW_VERSION="" # Additional PIP flags passed to all pip install commands except reinstalling pip itself ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ARG AIRFLOW_USE_UV="true" ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \ AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \ CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \ AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ UV_SYSTEM_PYTHON="true" \ UV_PROJECT_ENVIRONMENT="/usr/local" \ INSTALL_MYSQL_CLIENT="true" \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ INSTALL_MSSQL_CLIENT="true" \ INSTALL_POSTGRES_CLIENT="true" \ AIRFLOW_INSTALLATION_METHOD="." \ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ INCLUDE_PRE_RELEASE="true" \ CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \ CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON} RUN echo "Airflow version: ${AIRFLOW_VERSION}" # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here COPY --from=scripts common.sh install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/ # We are first creating a venv where all python packages and .so binaries needed by those are # installed. # Here we fix the versions so all subsequent commands will use the versions # from the sources # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` ARG AIRFLOW_PIP_VERSION=26.2.2 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" ARG AIRFLOW_UV_VERSION=0.13.6 ARG AIRFLOW_PREK_VERSION="0.5.7" # UV_LINK_MODE=copy is needed since we are using cache mounted from the host ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ UV_LINK_MODE=copy \ AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION} # The PATH is needed for python to find installed and cargo to build the wheels ENV PATH="/usr/python/bin:/root/.local/bin:${PATH}" # Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from # an incorrect architecture. ARG TARGETARCH # Value to be able to easily change cache id and therefore use a bare new cache ARG DEPENDENCY_CACHE_EPOCH="2" # Install useful command line tools in their own virtualenv so that they do not clash with # dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed # in the version specified above RUN bash /scripts/docker/install_packaging_tools.sh COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/ COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/ ARG UPGRADE_RANDOM_INDICATOR_STRING="" # Only preinstall locked third-party dependencies, with the same uv sync as the full installation. # A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still # installs workspace packages from the current checkout and retains its resolution fallback. Any other # failure would fail the full sync as well, so it fails the build here. RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \ if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \ INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \ fi # We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not # copying over stuff that is accidentally generated or that we do not need (such as egg-info) # if you want to add something that is missing and you expect to see it in the image you can # add it with ! in .dockerignore next to the airflow, test etc. directories there COPY . ${AIRFLOW_SOURCES}/ ARG VERSION_SUFFIX="" ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \ VERSION_SUFFIX=${VERSION_SUFFIX} # The goal of this line is to install the dependencies from the most current pyproject.toml from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. # Usually we will install versions based on the dependencies in pyproject.toml and upgraded only if needed. # But in cron job we will install latest versions matching pyproject.toml to see if there is no breaking change # and push the constraints if everything is successful RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ bash /scripts/docker/install_airflow_when_building_images.sh COPY --from=scripts install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/ ARG ADDITIONAL_PYTHON_DEPS="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \ bash /scripts/docker/install_packaging_tools.sh; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ bash /scripts/docker/install_additional_dependencies.sh; \ fi COPY --from=scripts entrypoint_ci.sh /entrypoint COPY --from=scripts entrypoint_exec.sh /entrypoint-exec RUN chmod a+x /entrypoint /entrypoint-exec # Install autocomplete for airflow and kubectl # hadolint ignore=SC2028 RUN if command -v airflow; then \ register-python-argcomplete airflow >> ~/.bashrc ; \ fi; \ echo "source /etc/bash_completion" >> ~/.bashrc ; \ echo 'export PS1="\[\033[1;36m\][Breeze:\$(python --version 2>&1 | cut -d\" \" -f2)]\[\033[0m\] \[\033[1;32m\]\u@\h\[\033[0m\]:\[\033[1;34m\]\w\[\033[0m\]\$ "' >> ~/.bashrc WORKDIR ${AIRFLOW_SOURCES} ARG BUILD_ID ARG COMMIT_SHA ARG AIRFLOW_IMAGE_DATE_CREATED ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${PATH}" \ GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" \ BUILD_ID=${BUILD_ID} \ COMMIT_SHA=${COMMIT_SHA} \ # When we enter the image, the /root/.cache is not mounted from temporary mount cache. # We do not want to share the cache from host to avoid all kinds of problems where cache # is different with different platforms / python versions. We want to have a clean cache # in the image - and in this case /root/.cache is on the same filesystem as the installed packages. # so we can go back to the default link mode being hardlink. UV_LINK_MODE=hardlink \ MYPY_FORCE_COLOR="true" # Link dumb-init for backwards compatibility (so that older images also work) RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init EXPOSE 8080 LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.module="airflow" \ org.apache.airflow.component="airflow" \ org.apache.airflow.image="airflow-ci" \ org.apache.airflow.version="${AIRFLOW_VERSION}" \ org.apache.airflow.python.version="${AIRFLOW_PYTHON_VERSION}" \ org.apache.airflow.uid="0" \ org.apache.airflow.gid="0" \ org.apache.airflow.build-id="${BUILD_ID}" \ org.apache.airflow.commit-sha="${COMMIT_SHA}" \ org.opencontainers.image.source="${AIRFLOW_IMAGE_REPOSITORY}" \ org.opencontainers.image.created="${AIRFLOW_IMAGE_DATE_CREATED}" \ org.opencontainers.image.authors="dev@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.source="https://github.com/apache/airflow" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow-ci-image" \ org.opencontainers.image.title="Continuous Integration Airflow Image" \ org.opencontainers.image.description="Installed Apache Airflow with Continuous Integration dependencies" ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD []