nprasad2077/airflow
1
0 Fork 0
mirror of https://github.com/apache/airflow.git synced 2026-11-04 07:12:48 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
cfbd70b66d11f5d464eb9c0ba926dd34304e110d
airflow/Dockerfile.ci
T
github-actions[bot]andJarek Potiuk cfbd70b66d [main] Upgrade important CI environment (#76278) 
* [main] CI: Upgrade important CI environment

* Ignore pydantic-ai's generic OpenAI-compatible provider in vendor drift checks

pydantic-ai-slim 2.63.0 adds a provider for any OpenAI-compatible endpoint. It needs a service name at construction and infer_model has no prefix for it, so a common.ai connection can never reach it and it is not a vendor the services page should list.

Generated-by: Claude Opus 5

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
2026-11-02 19:39:05 +01:00

2410 lines
102 KiB
Docker
Raw Blame History

# syntax=docker/dockerfile:1.4
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
#
# BASE_IMAGE: the Docker Hardened Image that Python comes from. Both the build stage and the final
# image are based on it, so Python is never compiled during the build.
#
# The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no
# registry credentials. The upstream image itself requires a "docker login dhi.io". It is pinned to a
# digest, because the mirror's tags move several times a day: "breeze ci upgrade" moves the pin once
# the full CI has passed on the new image.
#
# The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the
# default, but you can build a FIPS-compliant image by pointing the build at one:
# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.18-debian13-fips-dev"
ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.18-debian13-dev@sha256:cacda18986eefebc42b0c6c03565ec2d65a4507b6adff559e4daf0f1e2bfe338"
##############################################################################################
# This is the script image where we keep all inlined bash scripts needed in other segments
# We use BASE_IMAGE to make sure that the scripts are different for different platforms.
##############################################################################################
FROM ${BASE_IMAGE} as scripts
##############################################################################################
# Please DO NOT modify the inlined scripts manually. The content of those files will be
# replaced by prek automatically from the "scripts/docker/" folder.
# This is done in order to avoid problems with caching and file permissions and in order to
# make the PROD Dockerfile standalone
##############################################################################################
# The content below is automatically copied from scripts/docker/install_os_dependencies.sh
COPY <<"EOF" /install_os_dependencies.sh
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" != 1 ]]; then
echo
echo "ERROR! There should be 'runtime', 'ci' or 'dev' parameter passed as argument.".
echo
exit 1
fi
GOLANG_MAJOR_MINOR_VERSION=${GOLANG_MAJOR_MINOR_VERSION:-1.24.4}
TEMURIN_VERSION=${TEMURIN_VERSION:-11}
NODEJS_VERSION=${NODEJS_VERSION:-22.23.1}
PNPM_VERSION=${PNPM_VERSION:-10.28.1}
RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable}
RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened}
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-}
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
# The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as
# Debian packages under /usr - so ask the base image's Python where it lives.
PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')}
# Read before apt runs: a package depending on python3 can pull another Python version in.
BASE_PYTHON_MAJOR_MINOR=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")')
if [[ -n "${AIRFLOW_PYTHON_VERSION}" && "${AIRFLOW_PYTHON_VERSION}." != "${BASE_PYTHON_MAJOR_MINOR}."* ]]; then
echo
echo "ERROR! AIRFLOW_PYTHON_VERSION is ${AIRFLOW_PYTHON_VERSION}, but the base image has Python ${BASE_PYTHON_MAJOR_MINOR}."
echo "The hardened flavor takes Python from BASE_IMAGE: leave AIRFLOW_PYTHON_VERSION empty or pick a BASE_IMAGE with that Python."
echo
exit 1
fi
elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then
PYTHON_HOME=${PYTHON_HOME:-/usr/python}
else
echo
echo "ERROR! AIRFLOW_IMAGE_FLAVOR should be 'hardened' or 'legacy', not '${AIRFLOW_IMAGE_FLAVOR}'."
echo
exit 1
fi
if [[ "${1}" == "runtime" ]]; then
INSTALLATION_TYPE="RUNTIME"
elif [[ "${1}" == "dev" ]]; then
INSTALLATION_TYPE="DEV"
elif [[ "${1}" == "ci" ]]; then
INSTALLATION_TYPE="CI"
else
echo
echo "ERROR! Wrong argument. Passed ${1} and it should be one of 'runtime', 'ci' or 'dev'.".
echo
exit 1
fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
DEV_APT_DEPS="\
apt-transport-https \
apt-utils \
build-essential \
dirmngr \
freetds-bin \
freetds-dev \
git \
graphviz \
graphviz-dev \
gzip \
krb5-user \
ldap-utils \
libc6-dev \
libev-dev \
libev4 \
libffi-dev \
libgeos-dev \
libkrb5-dev \
libldap2-dev \
libleveldb-dev \
libleveldb1d \
libsasl2-2 \
libsasl2-dev \
libsasl2-modules \
libssl-dev \
libxmlsec1 \
libxmlsec1-dev \
locales \
openssh-client \
openssl \
pkg-config \
pkgconf \
sasl2-bin \
sqlite3 \
sudo \
tdsodbc \
unixodbc \
unixodbc-dev \
wget \
xz-utils \
zlib1g-dev \
"
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then
# Needed only to compile Python from sources - without them it builds, but silently
# lacks modules such as sqlite3, lzma or readline.
local debian_version
# shellcheck disable=SC1091
debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME")
if [[ "${debian_version}" == "bookworm" ]]; then
DEV_APT_DEPS+="lcov libbluetooth-dev libbz2-dev libdb-dev libgdbm-compat-dev libgdbm-dev \
liblzma-dev libncurses5-dev libreadline6-dev libsqlite3-dev libzstd-dev lzma lzma-dev tk-dev uuid-dev"
else
DEV_APT_DEPS+="lcov libbluetooth-dev libbz2-dev libdb-dev libgdbm-compat-dev libgdbm-dev \
liblzma-dev libncurses-dev libreadline-dev libsqlite3-dev libzstd-dev tk-dev uuid-dev"
fi
fi
export DEV_APT_DEPS
fi
}
function get_runtime_apt_deps() {
local debian_version
local debian_version_apt_deps
# Get debian version without installing lsb_release
# shellcheck disable=SC1091
debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
echo
echo "DEBIAN CODENAME: ${debian_version}"
echo
if [[ "${debian_version}" == "bookworm" ]]; then
debian_version_apt_deps="\
libffi8 \
libldap-2.5-0 \
libssl3 \
netcat-openbsd\
"
else
# trixie renamed the libraries that moved to a 64-bit time_t (libssl3 -> libssl3t64) and
# dropped the soname from the LDAP library package name.
debian_version_apt_deps="\
libffi8 \
libldap2 \
libssl3t64 \
netcat-openbsd\
"
fi
echo
echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"
echo
# libxmlsec1-openssl was added because libxmlsec1 ships no crypto engine of its own - the engines
# are separate packages - so the "xmlsec" module (pulled in by python3-saml) imported with
# "libxmlsec1-openssl.so.1: cannot open shared object file" without it.
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
RUNTIME_APT_DEPS="\
${debian_version_apt_deps} \
apt-transport-https \
apt-utils \
curl \
dumb-init \
freetds-bin \
git \
gnupg \
iputils-ping \
krb5-user \
ldap-utils \
libev4 \
libgeos-dev \
libsasl2-2 \
libsasl2-modules \
libxmlsec1 \
libxmlsec1-openssl \
locales \
openssh-client \
rsync \
sasl2-bin \
sqlite3 \
sudo \
tdsodbc \
unixodbc \
wget\
"
export RUNTIME_APT_DEPS
fi
}
function install_docker_cli() {
apt-get update
apt-get install ca-certificates curl
install -m 0755 -d /etc/apt/keyrings
curl -fsSL --retry 3 --retry-delay 5 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc
# shellcheck disable=SC1091
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get install -y --no-install-recommends docker-ce-cli
}
function keep_image_conffiles() {
# The hardened base images ship a modified /etc/debian_version, so any apt run that upgrades
# base-files stops at dpkg's interactive conffile prompt and fails a non-interactive build. Making
# dpkg keep the image's copy by default covers every apt run - this build's and those of images
# extending it - so no install command has to remember the flags.
mkdir -p /etc/dpkg/dpkg.cfg.d
printf '%s\n' force-confdef force-confold > /etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles
}
function prefer_hardened_apt_repository() {
# The hardened base images take their packages from dhi.io, but stock Debian's sources stay
# configured beside it and apt installs whichever repository offers the higher version. Debian's
# build of a package then replaces Docker's, and its dependencies no longer match the packages the
# image ships (docker-hardened-images/catalog#610). Ranking dhi.io above the default priority of
# 500 makes apt take Docker's build whenever dhi.io publishes the package, and fall back to Debian
# only when it does not.
# The priority stays below 1000 so the preference never forces a downgrade: apt still favors
# dhi.io's build of a package it publishes, but may take a newer version from another repository
# when a dependency requires it - for example PGDG's libpq5, which postgresql-client-18 needs at
# ">= 18.6" and dhi.io does not carry. The renumbered builds that sort below the image's own copy
# are handled by align_image_packages_with_hardened_repository instead.
# The preference covers every apt run - this build's and those of images extending it.
mkdir -p /etc/apt/preferences.d
printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' > /etc/apt/preferences.d/airflow-prefer-dhi
}
function align_image_packages_with_hardened_repository() {
# The dhi.io apt repository is live, so it moves on even under a base image pinned to a digest.
# Docker renumbered its builds ("14.2.0-19+dhi5" became "14.2.0-19+deb13u0+dhi5", which sorts
# lower) and the repository dropped the old numbers, while the image still ships them. Packages
# depending on the exact new version (gcc-14 on gcc-14-base, libgmp-dev on libgmp10) are then
# uninstallable, because apt will not downgrade the image's copy at a priority below 1000
# (docker-hardened-images/catalog#689). Only installed versions no repository offers any more
# ("local" ones) are moved to dhi.io's build, so newer packages from other repositories - such as
# PGDG's libpq5 - are left alone.
local package
local dhi_version
local -a downgrades=()
while read -r package; do
dhi_version=$(apt-cache madison "${package}" | awk -F' [|] ' '$3 ~ /dhi\.io/ {print $2; exit}')
if [[ -n "${dhi_version}" ]]; then
downgrades+=("${package}=${dhi_version}")
fi
done < <(apt list --installed 2>/dev/null | awk -F/ '/\[installed,local\]/ {print $1}')
if (( ${#downgrades[@]} )); then
echo "Aligning image packages with the dhi.io repository: ${downgrades[*]}"
apt-get install -y --no-install-recommends --allow-downgrades "${downgrades[@]}"
fi
}
function refresh_ldconfig_after_dpkg() {
# The hardened base images ship libc-bin without its dpkg trigger, so installing a shared library
# does not refresh the ldconfig cache and ctypes.util.find_library() - which reads that cache -
# cannot find it (for example pyenchant's "The 'enchant' C library was not found"). The hook covers
# every apt run - this build's and those of images extending it.
mkdir -p /etc/apt/apt.conf.d
echo 'DPkg::Post-Invoke { "if [ -x /sbin/ldconfig ]; then /sbin/ldconfig; fi"; };' \
> /etc/apt/apt.conf.d/99airflow-ldconfig
}
function restore_debian_base_files() {
# The hardened base images ship a minimal /etc, but Debian maintainer scripts assume the files
# a stock Debian has: sasl2-bin chowns its run directory to the "sasl" group from base-passwd,
# and tmux registers its shell with add-shell, which reads /etc/shells. The base-passwd package
# only ships the reference copies of the account files - update-passwd is what merges them
# into /etc.
# libpam-runtime generates the /etc/pam.d/common-* files that the PAM configs already in the
# image "@include" - without them "adduser --gecos" aborts with a PAM error from chfn.
# Debian Policy lets a maintainer script rely on any essential package without declaring it, and
# libgcrypt20 takes that up: its postinst runs a helper with a "#!/bin/dash" shebang. The hardened
# images ship /bin/sh as a symlink to bash and no dash at all, so configuring the package aborts
# with exit 127 and takes the whole apt transaction with it. Installing it first restores the
# assumption, and hands /bin/sh back to dash the way a stock Debian has it.
apt-get install -y --no-install-recommends dash
apt-get install -y --no-install-recommends base-passwd libpam-runtime
update-passwd
# The Debian 13 hardened images strip /etc/pam.d further: libpam-runtime and passwd are already
# installed there, so installing them generates nothing, and passwd, chpasswd and chfn abort with
# "pam_start() failed". Reinstalling passwd brings back its deleted PAM configs, and
# pam-auth-update - which treats the deleted common-* files as local changes - needs --force.
apt-get install -y --no-install-recommends --reinstall -o Dpkg::Options::=--force-confmiss passwd
if [[ ! -e /etc/pam.d/common-auth ]]; then
pam-auth-update --package --force
fi
if [[ ! -e /etc/shells ]]; then
printf '%s\n' "# /etc/shells: valid login shells" /bin/sh /bin/bash > /etc/shells
fi
align_perl_base_with_perl
}
function align_perl_base_with_perl() {
# The hardened images ship their own build of perl-base, but the hardened apt repository does not
# always publish the matching perl - while it still publishes packages that depend on perl, such
# as git. Debian's perl depends on exactly its own perl-base version, so apt refuses to install
# git at all. Moving perl-base to the version of the perl apt can install keeps the two in step;
# when the repository does publish a matching perl, the versions already agree and this does nothing.
local perl_version
local perl_base_version
perl_version=$(apt-cache policy perl | awk '/Candidate:/ {print $2}')
perl_base_version=$(dpkg-query -W -f='${Version}' perl-base 2>/dev/null || true)
if [[ -n "${perl_version}" && "${perl_version}" != "(none)" && -n "${perl_base_version}" \
&& "${perl_base_version}" != "${perl_version}" ]]; then
apt-get install -y --no-install-recommends --allow-downgrades "perl-base=${perl_version}"
fi
}
function install_debian_dev_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
apt-get install -yqq --no-install-recommends apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_DEV_APT_ENV?}
if [[ ${DEV_APT_COMMAND} != "" ]]; then
bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}"
fi
if [[ ${ADDITIONAL_DEV_APT_COMMAND} != "" ]]; then
bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}"
fi
apt-get update
local debian_version
local debian_version_apt_deps
# Get debian version without installing lsb_release
# shellcheck disable=SC1091
debian_version=$(. /etc/os-release; printf '%s\n' "$VERSION_CODENAME";)
echo
echo "DEBIAN CODENAME: ${debian_version}"
echo
# shellcheck disable=SC2086
apt-get install -y --no-install-recommends ${DEV_APT_DEPS}
}
function install_additional_dev_dependencies() {
if [[ "${ADDITIONAL_DEV_APT_DEPS=}" != "" ]]; then
# shellcheck disable=SC2086
apt-get install -y --no-install-recommends ${ADDITIONAL_DEV_APT_DEPS}
fi
}
function link_python() {
# Airflow images have always exposed Python under /usr/python - documentation, volume mounts and
# user customizations refer to that path - while the hardened base images ship it in /opt/python
# (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink.
if [[ ! -e /usr/python ]]; then
ln -sv "${PYTHON_HOME}" /usr/python
fi
# The hardened base images have no /usr/local tree at all
mkdir -p /usr/local/bin /usr/local/lib
# link python binaries to /usr/local/bin and /usr/python/bin with and without 3 suffix
# Links in /usr/local/bin are needed for tools that expect python to be there
# Links in /usr/python/bin are needed for tools that are detecting home of python installation including
# lib/site-packages. The /usr/python/bin should be first in PATH in order to help with the last part.
for dst in pip3 python3 python3-config; do
src="$(echo "${dst}" | tr -d 3)"
if [[ ! -e "/usr/python/bin/${dst}" ]]; then
continue
fi
echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
ln -sfv "/usr/python/bin/${dst}" "/usr/local/bin/${dst}"
for dir in /usr/local/bin /usr/python/bin; do
if [[ ! -e "${dir}/${src}" ]]; then
echo "Creating ${src} - > ${dst} link in ${dir}"
ln -sv "${dir}/${dst}" "${dir}/${src}"
fi
done
done
# A Python installed under /usr already has its libraries where the dynamic linker looks, and
# linking all of /usr/lib into /usr/local/lib would only shadow the system libraries.
if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then
for dst in /usr/python/lib/*
do
src="/usr/local/lib/$(basename "${dst}")"
if [[ -e "${src}" ]]; then
rm -rf "${src}"
fi
echo "Linking ${dst} to ${src}"
ln -sv "${dst}" "${src}"
done
fi
ldconfig
}
function restore_thread_stack_size() {
# The hardened base images build Python with -DTHREAD_STACK_SIZE=0x100000, so every thread Python
# starts gets a 1 MiB stack instead of glibc's default (the 8 MiB "ulimit -s" the previous images
# used). On Python 3.12 and 3.13 the C recursion guard is a fixed depth count sized for the larger
# stack, so deeply nested input - e.g. a JSON body parsed in a web server worker thread - overflows
# the stack and kills the process instead of raising RecursionError. A .pth file runs at every
# interpreter start-up without taking the sitecustomize module name users may already rely on.
local site_packages
site_packages="$(/usr/python/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
echo "Restoring the 8 MiB default thread stack size in ${site_packages}"
echo "import threading; threading.stack_size(8 * 1024 * 1024)" > "${site_packages}/airflow-thread-stack-size.pth"
}
function compile_python_stdlib() {
# The hardened base images ship the standard library with no .pyc files at all. Python then misses
# the bytecode cache on every stdlib import, and in a read-only or non-writable directory it cannot
# create one - each miss leaves a negative dentry in the kernel, which grows without bound in a
# long-running container and can exhaust memory on the host. Compiling the standard library here
# restores what the previously compiled-in-image Python shipped.
# See https://github.com/apache/airflow/pull/58944 and https://lwn.net/Articles/814535/
local stdlib
stdlib="$(/usr/python/bin/python -c 'import sysconfig; print(sysconfig.get_paths()["stdlib"])')"
echo "Compiling Python standard library in ${stdlib}"
# compileall exits non-zero when any file fails to compile, and the standard library ships files
# that are meant not to compile (deliberately broken syntax used by the test suite).
/usr/python/bin/python -m compileall -q -j "$(nproc)" -o 0 -o 1 -o 2 "${stdlib}" || true
}
function hold_hardened_python() {
# The Debian 13 hardened images install their Python as packages that conflict with Debian's own
# python3. A package depending on python3 - for example NodeSource's nodejs - therefore makes apt
# remove the image's Python and install Debian's, without asking. Holding the packages turns that
# into an apt error instead, in this build and in images extending it.
local packages
packages=$(dpkg-query -W -f='${db:Status-Abbrev} ${Package}\n' 'python-3.*' 'libpython-3.*' 2>/dev/null \
| awk '$1 == "ii" {print $2}' || true)
if [[ -n "${packages}" ]]; then
# shellcheck disable=SC2086
apt-mark hold ${packages}
fi
}
function check_no_system_python() {
# Python - from the hardened base image or compiled from sources - must stay the only Python in the
# image. A system Python pulled in as a dependency of an apt package shares its shared libraries
# with ours and leads to errors such as:
# /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add
# Debian names its Python libraries "libpython3.13". The trixie hardened images register their own
# Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match.
# Debian 13 hardened images package their Python as "python-3.13" and serve other versions from
# their apt repository, so a package depending on python3 can also pull in a hardened Python of a
# different version than the image's own.
local other_hardened_python=""
if [[ -n "${BASE_PYTHON_MAJOR_MINOR=}" ]]; then
other_hardened_python=$(dpkg -l | awk '/^ii (lib)?python-3\.[0-9]+/ {print $2}' \
| grep -v -E "python-${BASE_PYTHON_MAJOR_MINOR//./\\.}(-|:|$)" || true)
fi
if [[ -n "${other_hardened_python}" ]]; then
echo
echo "ERROR! A Python other than the image's ${BASE_PYTHON_MAJOR_MINOR} was installed: ${other_hardened_python//$'\n'/ }"
echo
apt-get install -yqq aptitude >/dev/null
aptitude why "$(echo "${other_hardened_python}" | head -1 | cut -d: -f1)"
echo
exit 1
fi
if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then
echo
echo "ERROR! System python is installed by one of the previous steps"
echo
echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:"
echo
apt-get install -yqq aptitude >/dev/null
aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')"
echo
exit 1
else
echo
echo "GOOD! System python is not installed - OK"
echo
fi
}
function install_cosign() {
local arch
arch="$(dpkg --print-architecture)"
declare -A cosign_sha256s=(
# https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign_checksums.txt
[amd64]="db15cc99e6e4837daabab023742aaddc3841ce57f193d11b7c3e06c8003642b2"
[arm64]="d098f3168ae4b3aa70b4ca78947329b953272b487727d1722cb3cb098a1a20ab"
)
local cosign_sha256="${cosign_sha256s[${arch}]}"
if [[ -z "${cosign_sha256}" ]]; then
echo "Unsupported architecture for cosign: ${arch}"
exit 1
fi
curl -fsSL --retry 3 --retry-delay 5 \
"https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" \
-o /tmp/cosign
echo "${cosign_sha256} /tmp/cosign" | sha256sum --check
chmod +x /tmp/cosign
}
function resolve_python_version() {
if [[ -z "${AIRFLOW_PYTHON_VERSION}" ]]; then
echo
echo "ERROR! The legacy flavor compiles Python from sources, so AIRFLOW_PYTHON_VERSION has to name it:"
echo "MAJOR.MINOR for the latest release of that line, or MAJOR.MINOR.PATCH."
echo
exit 1
fi
if [[ ! "${AIRFLOW_PYTHON_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then
return
fi
# A release directory appears on python.org with its first pre-release, so take the newest one
# that already holds the final tarball.
local candidate
for candidate in $(wget --tries=3 --waitretry=5 -qO- https://www.python.org/ftp/python/ \
| grep -oE "href=\"${AIRFLOW_PYTHON_VERSION//./\\.}\.[0-9]+/\"" \
| grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r); do
if wget --tries=3 --waitretry=5 -q --spider \
"https://www.python.org/ftp/python/${candidate}/Python-${candidate}.tar.xz"; then
echo "Python ${AIRFLOW_PYTHON_VERSION} resolved to its latest release ${candidate}"
AIRFLOW_PYTHON_VERSION="${candidate}"
return
fi
done
echo
echo "ERROR! No release of Python ${AIRFLOW_PYTHON_VERSION} found on python.org."
echo
exit 1
}
function install_python() {
# Only the legacy image flavor compiles Python - the hardened flavor gets it from the base image.
resolve_python_version
wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz"
local major_minor_version
major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}"
echo "Verifying Python ${AIRFLOW_PYTHON_VERSION} (${major_minor_version})"
# Sigstore verification (PEP 761)
declare -A sigstore_identities=(
# https://peps.python.org/pep-0664/#release-manager-and-crew
[3.11]="pablogsal@python.org"
# https://peps.python.org/pep-0693/#release-manager-and-crew
[3.12]="thomas@python.org"
# https://peps.python.org/pep-0719/#release-manager-and-crew
[3.13]="thomas@python.org"
# https://peps.python.org/pep-0745/#release-manager-and-crew
[3.14]="hugo@python.org"
# https://peps.python.org/pep-0790/#release-manager-and-crew
[3.15]="hugo@python.org"
)
declare -A sigstore_issuers=(
[3.11]="https://accounts.google.com"
[3.12]="https://accounts.google.com"
[3.13]="https://accounts.google.com"
[3.14]="https://github.com/login/oauth"
[3.15]="https://github.com/login/oauth"
)
wget --tries=3 --waitretry=5 -O python.tar.xz.sigstore \
"https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz.sigstore"
install_cosign
local identity="${sigstore_identities[${major_minor_version}]}"
local issuer="${sigstore_issuers[${major_minor_version}]}"
/tmp/cosign verify-blob \
--bundle python.tar.xz.sigstore \
--certificate-identity "${identity}" \
--certificate-oidc-issuer "${issuer}" \
python.tar.xz
rm -f python.tar.xz.sigstore /tmp/cosign
mkdir -p /usr/src/python
tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz
rm python.tar.xz
cd /usr/src/python
arch="$(dpkg --print-architecture)"; arch="${arch##*-}"
gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"
EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"
EXTRA_CFLAGS="${EXTRA_CFLAGS:-} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer";
LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"
LDFLAGS="${LDFLAGS:--Wl},--strip-all"
local build_log
build_log=$(mktemp)
echo "Building Python ${AIRFLOW_PYTHON_VERSION} from source..."
if ! (
./configure --enable-optimizations --prefix=/usr/python/ --with-ensurepip --build="$gnuArch" \
--enable-loadable-sqlite-extensions --enable-option-checking=fatal \
--enable-shared --with-lto && \
make -s -j "$(nproc)" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python && \
make -s -j "$(nproc)" install
) > "${build_log}" 2>&1; then
echo
echo "ERROR! Python build failed. Build output:"
echo
cat "${build_log}"
rm -f "${build_log}"
exit 1
fi
rm -f "${build_log}"
# CPython builds without the optional modules whose -dev packages are missing, so check them here
# rather than finding out from a broken image.
/usr/python/bin/python3 -c "import bz2, ctypes, dbm.gnu, lzma, readline, sqlite3, ssl, uuid, zlib"
cd /
rm -rf /usr/src/python
find /usr/python -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \( -type f -a \( -name 'libpython*.a' \) \) \
\) -exec rm -rf '{}' +
link_python
}
function install_debian_runtime_dependencies() {
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
keep_image_conffiles
prefer_hardened_apt_repository
refresh_ldconfig_after_dpkg
fi
apt-get update
apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
align_image_packages_with_hardened_repository
restore_debian_base_files
fi
apt-get install -y --no-install-recommends wget curl gnupg2 ca-certificates
# shellcheck disable=SC2086
export ${ADDITIONAL_RUNTIME_APT_ENV?}
if [[ "${RUNTIME_APT_COMMAND}" != "" ]]; then
bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}"
fi
if [[ "${ADDITIONAL_RUNTIME_APT_COMMAND}" != "" ]]; then
bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}"
fi
apt-get update
# shellcheck disable=SC2086
apt-get install -y --no-install-recommends ${RUNTIME_APT_DEPS} ${ADDITIONAL_RUNTIME_APT_DEPS}
apt-get autoremove -yqq --purge
apt-get clean
check_no_system_python
link_python
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
hold_hardened_python
restore_thread_stack_size
compile_python_stdlib
fi
rm -rf /var/lib/apt/lists/* /var/log/*
}
function install_golang() {
curl --retry 3 --retry-delay 5 "https://dl.google.com/go/go${GOLANG_MAJOR_MINOR_VERSION}.linux-$(dpkg --print-architecture).tar.gz" -o "go${GOLANG_MAJOR_MINOR_VERSION}.linux.tar.gz"
rm -rf /usr/local/go && tar -C /usr/local -xzf go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
rm -f go"${GOLANG_MAJOR_MINOR_VERSION}".linux.tar.gz
}
function install_jdk() {
# Install Eclipse Temurin JDK from the Adoptium apt repository (https://adoptium.net/installation/linux/).
apt-get update -qq
apt-get install -y --no-install-recommends wget gnupg apt-transport-https ca-certificates
mkdir -p /etc/apt/keyrings
wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public \
| tee /etc/apt/keyrings/adoptium.asc > /dev/null
# shellcheck disable=SC1091
DISTRO_CODENAME=$(. /etc/os-release; echo "${VERSION_CODENAME}")
echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] \
https://packages.adoptium.net/artifactory/deb ${DISTRO_CODENAME} main" \
| tee /etc/apt/sources.list.d/adoptium.list > /dev/null
apt-get update -qq
apt-get install -y --no-install-recommends "temurin-${TEMURIN_VERSION}-jdk"
apt-get clean
rm -rf /var/lib/apt/lists/*
}
function install_nodejs() {
local arch
arch="$(dpkg --print-architecture)"
declare -A nodejs_targets=(
[amd64]="linux-x64"
[arm64]="linux-arm64"
)
declare -A nodejs_sha256s=(
# https://nodejs.org/dist/v${NODEJS_VERSION}/SHASUMS256.txt
[amd64]="9749e988f437343b7fa832c69ded82a312e41a03116d766797ac14f6f9eee578"
[arm64]="0294e8b915ab75f92c7513d2fcb830ae06e10684e6c603e99a87dbf8835389c1"
)
local target="${nodejs_targets[${arch}]}"
local nodejs_sha256="${nodejs_sha256s[${arch}]}"
if [[ -z "${target}" ]]; then
echo "Unsupported architecture for nodejs: ${arch}"
exit 1
fi
curl --retry 3 --retry-delay 5 \
"https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-${target}.tar.xz" \
-o /tmp/nodejs.tar.xz
echo "${nodejs_sha256} /tmp/nodejs.tar.xz" | sha256sum --check
tar -xJf /tmp/nodejs.tar.xz --strip-components=1 -C /usr/local --no-same-owner
rm -f /tmp/nodejs.tar.xz
corepack enable --install-directory /usr/local/bin
# corepack enable only writes shims; prepare downloads and caches the pnpm binary into the
# image so it is available offline and matches the version ts-sdk pins.
corepack prepare "pnpm@${PNPM_VERSION}" --activate
}
function install_rustup() {
local arch
arch="$(dpkg --print-architecture)"
declare -A rustup_targets=(
[amd64]="x86_64-unknown-linux-gnu"
[arm64]="aarch64-unknown-linux-gnu"
)
declare -A rustup_sha256s=(
# https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/{target}/rustup-init.sha256
[amd64]="4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10"
[arm64]="9732d6c5e2a098d3521fca8145d826ae0aaa067ef2385ead08e6feac88fa5792"
)
local target="${rustup_targets[${arch}]}"
local rustup_sha256="${rustup_sha256s[${arch}]}"
if [[ -z "${target}" ]]; then
echo "Unsupported architecture for rustup: ${arch}"
exit 1
fi
curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-delay 5 \
"https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${target}/rustup-init" \
-o /tmp/rustup-init
echo "${rustup_sha256} /tmp/rustup-init" | sha256sum --check
chmod +x /tmp/rustup-init
# Building wheels from source needs only rustc and cargo. The default profile also adds
# rust-docs, clippy and rustfmt, which add tens of thousands of files to the image.
/tmp/rustup-init -y --profile minimal --default-toolchain "${RUSTUP_DEFAULT_TOOLCHAIN}"
rm -f /tmp/rustup-init
}
function apt_clean() {
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
rm -rf /var/lib/apt/lists/* /var/log/*
}
if [[ "${INSTALLATION_TYPE}" == "RUNTIME" ]]; then
get_runtime_apt_deps
install_debian_runtime_dependencies
install_docker_cli
apt_clean
else
get_dev_apt_deps
install_debian_dev_dependencies
check_no_system_python
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
link_python
hold_hardened_python
restore_thread_stack_size
compile_python_stdlib
else
install_python
fi
install_additional_dev_dependencies
install_rustup
if [[ "${INSTALLATION_TYPE}" == "CI" ]]; then
install_golang
install_jdk
install_nodejs
fi
install_docker_cli
apt_clean
fi
EOF
# The content below is automatically copied from scripts/docker/install_mysql.sh
COPY <<"EOF" /install_mysql.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
set -euo pipefail
common::get_colors
declare -a packages
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
: "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then
echo
echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}"
echo
exit 1
fi
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then
echo
echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}"
echo
exit 1
fi
if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
echo
echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}"
echo
echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys,"
echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client."
echo ""
echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling"
echo "with fixes and re-signing the packages which lasts few days"
echo "See https://bugs.mysql.com/bug.php?id=113432 for more details."
echo "As a community we are not able to support this broken packaging practice from Oracle"
echo "Feel free however to install MySQL drivers on your own as extension of the image."
echo
exit 1
fi
retry() {
local retries=3
local count=0
# adding delay of 10 seconds
local delay=10
until "$@"; do
exit_code=$?
count=$((count + 1))
if [[ $count -lt $retries ]]; then
echo "Command failed. Attempt $count/$retries. Retrying in ${delay}s..."
sleep $delay
else
echo "Command failed after $retries attempts."
return $exit_code
fi
done
}
install_mariadb_client() {
# https://mariadb.org/about/#maintenance-policy
# MariaDB publishes each LTS series only for the Debian releases current when it came out - 10.11
# has no trixie repository and 11.8 is the first LTS that does.
local mariadb_lts_version="10.11"
if [[ "$(common::debian_codename)" != "bookworm" ]]; then
mariadb_lts_version="11.8"
fi
# List of compatible package Oracle MySQL -> MariaDB:
# `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
# `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`
# `libmysqlclient-dev` -> `libmariadb-dev-compat`
#
# Different naming against Debian repo which we used before
# that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:
# `libmariadb-dev` -> `libmariadb-dev-compat`
# `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)
if [[ "${1}" == "dev" ]]; then
packages=("libmariadb-dev-compat" "mariadb-client")
elif [[ "${1}" == "prod" ]]; then
packages=("libmariadb3-compat" "mariadb-client")
else
echo
echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
if [[ "${mariadb_lts_version%%.*}" -ge 11 ]]; then
# From 11.0 mariadb-client no longer ships the "mysql" named commands
packages+=("mariadb-client-compat")
fi
common::import_trusted_gpg "0xF1656F24C74CD1D8" "mariadb"
echo
echo "${COLOR_BLUE}Installing MariaDB client version ${mariadb_lts_version}: ${1}${COLOR_RESET}"
echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"
echo
echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${mariadb_lts_version}/repo/debian/ $(common::debian_codename) main" > \
/etc/apt/sources.list.d/mariadb.list
# Make sure that dependencies from MariaDB repo are preferred over Debian dependencies
printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb
retry apt-get update
retry apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
apt-get clean && rm -rf /var/lib/apt/lists/*
}
if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
install_mariadb_client "${@}"
fi
EOF
# The content below is automatically copied from scripts/docker/install_mssql.sh
COPY <<"EOF" /install_mssql.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
set -euo pipefail
common::get_colors
declare -a packages
: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
function install_mssql_client() {
# Install MsSQL client from Microsoft repositories
if [[ ${INSTALL_MSSQL_CLIENT:="true"} != "true" ]]; then
echo
echo "${COLOR_BLUE}Skip installing mssql client${COLOR_RESET}"
echo
return
fi
packages=("msodbcsql18")
common::import_trusted_gpg "EB3E94ADBE1229CF" "microsoft"
# The Debian 13 repository is signed with Microsoft's newer key, the older releases with the one above.
common::import_trusted_gpg "EE4D7792F748182B" "microsoft-2025"
echo
echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}"
echo
echo "deb [arch=amd64,arm64] https://packages.microsoft.com/debian/$(common::debian_release)/prod $(common::debian_codename) main" > \
/etc/apt/sources.list.d/mssql-release.list &&
mkdir -p /opt/microsoft/msodbcsql18 &&
touch /opt/microsoft/msodbcsql18/ACCEPT_EULA &&
apt-get update -yqq &&
apt-get upgrade -yqq &&
apt-get -yqq install --no-install-recommends "${packages[@]}" &&
apt-get autoremove -yqq --purge &&
apt-get clean &&
rm -rf /var/lib/apt/lists/*
}
install_mssql_client "${@}"
EOF
# The content below is automatically copied from scripts/docker/install_postgres.sh
COPY <<"EOF" /install_postgres.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
set -euo pipefail
common::get_colors
declare -a packages
: "${INSTALL_POSTGRES_CLIENT:?Should be true or false}"
install_postgres_client() {
echo
echo "${COLOR_BLUE}Installing postgres client${COLOR_RESET}"
echo
if [[ "${1}" == "dev" ]]; then
packages=("libpq-dev" "postgresql-client")
elif [[ "${1}" == "prod" ]]; then
packages=("postgresql-client")
else
echo
echo "Specify either prod or dev"
echo
exit 1
fi
common::import_trusted_gpg "7FCC7D46ACCC4CF8" "postgres"
echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \
/etc/apt/sources.list.d/pgdg.list
# The hardened image's airflow-prefer-dhi pin outranks Debian's default, so apt would otherwise keep
# dhi.io's libpq (e.g. 17.11) and postgresql-client (which needs a matching, newer libpq5) could not
# install. PostgreSQL's own repository is the authority for its packages - rank it above the dhi.io
# preference so apt takes libpq from PGDG.
mkdir -p /etc/apt/preferences.d
printf 'Package: *\nPin: origin "apt.postgresql.org"\nPin-Priority: 1001\n' > /etc/apt/preferences.d/airflow-prefer-pgdg
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
apt-get clean && rm -rf /var/lib/apt/lists/*
}
if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then
install_postgres_client "${@}"
fi
EOF
# The content below is automatically copied from scripts/docker/install_packaging_tools.sh
COPY <<"EOF" /install_packaging_tools.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
common::get_colors
common::get_packaging_tool
common::show_packaging_tool_version_and_location
common::install_packaging_tools
EOF
# The content below is automatically copied from scripts/docker/common.sh
COPY <<"EOF" /common.sh
#!/usr/bin/env bash
set -euo pipefail
function common::get_colors() {
COLOR_BLUE=$'\e[34m'
COLOR_GREEN=$'\e[32m'
COLOR_RED=$'\e[31m'
COLOR_RESET=$'\e[0m'
COLOR_YELLOW=$'\e[33m'
export COLOR_BLUE
export COLOR_GREEN
export COLOR_RED
export COLOR_RESET
export COLOR_YELLOW
}
function common::get_packaging_tool() {
: "${AIRFLOW_USE_UV:?Should be set}"
## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN
## `scripts/in_container/_in_container_utils.sh`
if [[ ${AIRFLOW_USE_UV} == "true" ]]; then
echo
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
echo
export PACKAGING_TOOL="uv"
export PACKAGING_TOOL_CMD="uv pip"
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -f "./pyproject.toml" ]]; then
# for uv only install dev group when we install from sources
export EXTRA_INSTALL_FLAGS="--group=dev --no-binary lxml --no-binary xmlsec"
else
export EXTRA_INSTALL_FLAGS="--no-binary lxml --no-binary xmlsec"
fi
export EXTRA_UNINSTALL_FLAGS=""
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --resolution highest"
export UPGRADE_IF_NEEDED="--upgrade"
UV_CONCURRENT_DOWNLOADS=$(nproc --all)
export UV_CONCURRENT_DOWNLOADS
if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --prerelease if-necessary"
fi
else
echo
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
echo
export PACKAGING_TOOL="pip"
export PACKAGING_TOOL_CMD="pip"
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
export EXTRA_INSTALL_FLAGS="--root-user-action ignore --no-binary lxml,xmlsec"
export EXTRA_UNINSTALL_FLAGS="--yes"
export UPGRADE_TO_HIGHEST_RESOLUTION="--upgrade --upgrade-strategy eager"
export UPGRADE_IF_NEEDED="--upgrade --upgrade-strategy only-if-needed"
if [[ ${INCLUDE_PRE_RELEASE=} == "true" ]]; then
EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS} --pre"
fi
fi
}
function common::get_airflow_version_specification() {
if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=}
&& -n ${AIRFLOW_VERSION}
&& ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then
AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}"
fi
}
function common::get_constraints_location() {
# When installing from sources without upgrade, generate constraints from uv.lock
if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." && -z "${UPGRADE_RANDOM_INDICATOR_STRING=}" ]]; then
echo
echo "${COLOR_BLUE}Installing from sources with uv.lock - generating constraints from uv.lock${COLOR_RESET}"
echo
uv export --frozen --no-hashes --no-emit-project --no-editable --no-header \
--no-annotate > "${HOME}/constraints.txt" 2>/dev/null || true
return
fi
# auto-detect Airflow-constraint reference and location
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* || ${AIRFLOW_VERSION} =~ v?3.* ]]; then
AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}
else
AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}
fi
fi
if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then
local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"
local python_version
python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"
fi
if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then
echo
echo "${COLOR_BLUE}Downloading constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
echo
if ! curl -sSf -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}"; then
echo
echo "${COLOR_YELLOW}Constraints file not found at ${AIRFLOW_CONSTRAINTS_LOCATION} (new Python version being bootstrapped?).${COLOR_RESET}"
echo "${COLOR_YELLOW}Falling back to no-constraints installation.${COLOR_RESET}"
echo
AIRFLOW_CONSTRAINTS_LOCATION=""
# Create an empty constraints file so --constraint flag still works
touch "${HOME}/constraints.txt"
fi
else
echo
echo "${COLOR_BLUE}Copying constraints from ${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
echo
cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"
fi
}
function common::show_packaging_tool_version_and_location() {
echo "PATH=${PATH}"
echo "Installed pip: $(pip --version): $(command -v pip)"
if [[ ${PACKAGING_TOOL} == "pip" ]]; then
echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}"
else
echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}"
echo "Installed uv: $(uv --version 2>/dev/null || echo "Not installed yet"): $(command -v uv 2>/dev/null)"
fi
}
function common::install_packaging_tools() {
: "${AIRFLOW_USE_UV:?Should be set}"
if [[ "${VIRTUAL_ENV=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"
echo
else
echo
echo "${COLOR_BLUE}Checking packaging tools for system Python installation: $(command -v python)${COLOR_RESET}"
echo
fi
if [[ ${AIRFLOW_PIP_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest pip version${COLOR_RESET}"
echo
pip install --root-user-action ignore --disable-pip-version-check --upgrade pip
elif [[ ! ${AIRFLOW_PIP_VERSION} =~ ^[0-9].* ]]; then
echo
echo "${COLOR_BLUE}Installing pip version from spec ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
else
local installed_pip_version
installed_pip_version=$(python -c 'from importlib.metadata import version; print(version("pip"))')
if [[ ${installed_pip_version} != "${AIRFLOW_PIP_VERSION}" ]]; then
echo
echo "${COLOR_BLUE}(Re)Installing pip version: ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
fi
fi
if [[ ${AIRFLOW_UV_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest uv version${COLOR_RESET}"
echo
pip install --root-user-action ignore --disable-pip-version-check --upgrade uv
elif [[ ! ${AIRFLOW_UV_VERSION} =~ ^[0-9].* ]]; then
echo
echo "${COLOR_BLUE}Installing uv version from spec ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}"
else
local installed_uv_version
installed_uv_version=$(python -c 'from importlib.metadata import version; print(version("uv"))' 2>/dev/null || echo "Not installed yet")
if [[ ${installed_uv_version} != "${AIRFLOW_UV_VERSION}" ]]; then
echo
echo "${COLOR_BLUE}(Re)Installing uv version: ${AIRFLOW_UV_VERSION}${COLOR_RESET}"
echo
# shellcheck disable=SC2086
pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"
fi
fi
if [[ ${AIRFLOW_PREK_VERSION=} == "" ]]; then
echo
echo "${COLOR_BLUE}Installing latest prek, uv${COLOR_RESET}"
echo
uv tool install prek --with uv
# make sure that the venv/user in .local exists
mkdir -p "${HOME}/.local/bin"
else
echo
echo "${COLOR_BLUE}Installing predefined versions of prek, uv:${COLOR_RESET}"
echo "${COLOR_BLUE}prek(${AIRFLOW_PREK_VERSION}) uv(${AIRFLOW_UV_VERSION})${COLOR_RESET}"
echo
uv tool install "prek==${AIRFLOW_PREK_VERSION}" --with "uv==${AIRFLOW_UV_VERSION}"
# make sure that the venv/user in .local exists
mkdir -p "${HOME}/.local/bin"
fi
}
function common::debian_codename() {
# shellcheck disable=SC1091
(. /etc/os-release; printf '%s\n' "${VERSION_CODENAME}")
}
function common::debian_release() {
# shellcheck disable=SC1091
(. /etc/os-release; printf '%s\n' "${VERSION_ID}")
}
function common::import_trusted_gpg() {
common::get_colors
local key=${1:?${COLOR_RED}First argument expects OpenPGP Key ID${COLOR_RESET}}
local name=${2:?${COLOR_RED}Second argument expected trust storage name${COLOR_RESET}}
local key_file="/scripts/docker/keys/${name}.asc"
echo "${COLOR_BLUE}Installing GPG public key ${key} from ${key_file}${COLOR_RESET}"
gpg --dearmor < "${key_file}" > "/etc/apt/trusted.gpg.d/${name}.gpg"
}
EOF
# The content below is automatically copied from scripts/docker/install_airflow_when_building_images.sh
COPY <<"EOF" /install_airflow_when_building_images.sh
#!/usr/bin/env bash
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
function install_from_sources() {
local extra_sync_flags
extra_sync_flags=""
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
echo
echo "${COLOR_RED}We only support uv not pip installation for upgrading dependencies!.${COLOR_RESET}"
echo
exit 1
fi
set +x
echo
echo "${COLOR_BLUE}Attempting to upgrade all packages to highest versions.${COLOR_RESET}"
echo
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
set -x
uv sync --all-packages --resolution highest --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
echo
exit 1
fi
echo
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies not reflected in uv.lock.${COLOR_RESET}"
echo
echo "${COLOR_BLUE}Falling back to re-resolving dependencies (uv sync without --frozen).${COLOR_RESET}"
echo
set -x
uv sync --all-packages --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
set +x
fi
fi
}
function install_from_external_spec() {
local installation_command_flags
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "apache-airflow" ]]; then
installation_command_flags="apache-airflow[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
else
echo
echo "${COLOR_RED}The '${AIRFLOW_INSTALLATION_METHOD}' installation method is not supported${COLOR_RESET}"
echo
echo "${COLOR_YELLOW}Supported methods are ('.', 'apache-airflow')${COLOR_RESET}"
echo
exit 1
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Remove airflow and all provider distributions installed before potentially${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true
set +x
echo
echo "${COLOR_BLUE}Installing all packages with highest resolutions. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
else
echo
echo "${COLOR_BLUE}Installing all packages with constraints. Installation method: ${AIRFLOW_INSTALLATION_METHOD}${COLOR_RESET}"
echo
set -x
if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags} --constraint "${HOME}/constraints.txt"; then
set +x
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because constraints installation failed and fallback is disabled.${COLOR_RESET}"
echo
exit 1
fi
echo
echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies conflicting with constraints.${COLOR_RESET}"
echo
echo "${COLOR_BLUE}Falling back to no-constraints installation.${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} ${ADDITIONAL_PIP_INSTALL_FLAGS} ${installation_command_flags}
set +x
fi
fi
}
function install_airflow_when_building_images() {
# Remove mysql from extras if client is not going to be installed
if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
echo "${COLOR_YELLOW}MYSQL client installation is disabled. Extra 'mysql' installations were therefore omitted.${COLOR_RESET}"
fi
# Remove postgres from extras if client is not going to be installed
if [[ ${INSTALL_POSTGRES_CLIENT} != "true" ]]; then
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/postgres,}
echo "${COLOR_YELLOW}Postgres client installation is disabled. Extra 'postgres' installations were therefore omitted.${COLOR_RESET}"
fi
# Determine the installation_command_flags based on AIRFLOW_INSTALLATION_METHOD method
if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then
install_from_sources
else
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
pip check
}
common::get_colors
common::get_packaging_tool
common::get_airflow_version_specification
common::get_constraints_location
common::show_packaging_tool_version_and_location
install_airflow_when_building_images
EOF
# The content below is automatically copied from scripts/docker/install_additional_dependencies.sh
COPY <<"EOF" /install_additional_dependencies.sh
#!/usr/bin/env bash
set -euo pipefail
: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
function install_additional_dependencies() {
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_TO_HIGHEST_RESOLUTION} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
pip check
else
echo
echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${UPGRADE_IF_NEEDED} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
set +x
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
pip check
fi
}
common::get_colors
common::get_packaging_tool
common::get_airflow_version_specification
common::get_constraints_location
common::show_packaging_tool_version_and_location
install_additional_dependencies
EOF
# The content below is automatically copied from scripts/docker/entrypoint_ci.sh
COPY <<"EOF" /entrypoint_ci.sh
#!/usr/bin/env bash
function set_verbose() {
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
set -x
else
set +x
fi
}
set_verbose
. "${AIRFLOW_SOURCES:-/opt/airflow}"/scripts/in_container/_in_container_script_init.sh
LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6"
export LD_PRELOAD
chmod 1777 /tmp
AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd)
PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.11}
export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
mkdir "${AIRFLOW_HOME}/sqlite" -p || true
ASSET_COMPILATION_WAIT_MULTIPLIER=${ASSET_COMPILATION_WAIT_MULTIPLIER:=1}
if [[ "${CI=}" == "true" ]]; then
export COLUMNS="202"
fi
. "${IN_CONTAINER_DIR}/check_connectivity.sh"
function wait_for_asset_compilation() {
if [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; then
echo
echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
echo
local counter=0
while [[ -f "${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock" ]]; do
if (( counter % 5 == 2 )); then
echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
fi
sleep 1
((counter=counter+1))
if [[ ${counter} == 30*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then
echo
echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
echo """
If it does not complete soon, you might want to stop it and remove file lock:
* press Ctrl-C
* run 'rm ${AIRFLOW_SOURCES}/.build/ui/.asset_compile.lock'
"""
fi
if [[ ${counter} == 60*$ASSET_COMPILATION_WAIT_MULTIPLIER ]]; then
echo
echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
echo "${COLOR_RED}refer to dev/breeze/doc/04_troubleshooting.rst for resolution steps.${COLOR_RED}"
echo
exit 1
fi
done
fi
if [ -f "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out" ]; then
echo
echo "${COLOR_RED}The asset compilation failed. Exiting.${COLOR_RESET}"
echo
cat "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out"
rm "${AIRFLOW_SOURCES}/.build/ui/asset_compile.out"
echo
exit 1
fi
}
function environment_initialization() {
if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then
return
fi
echo
echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}"
echo
echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}"
echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}"
echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}"
if [[ ${BACKEND=} == "postgres" ]]; then
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Postgres: ${POSTGRES_VERSION}"
elif [[ ${BACKEND=} == "mysql" ]]; then
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
elif [[ ${BACKEND=} == "sqlite" ]]; then
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
elif [[ ${BACKEND=} == "custom" ]]; then
local _conn_url="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-}"
local _masked_url
_masked_url=$(echo "${_conn_url}" | sed -E 's|://([^:]+):([^@]+)@|://\1:***@|')
echo " * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Custom (${_masked_url})"
fi
echo
if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then
echo
echo "${COLOR_BLUE}Forcing scheduler/standalone_dag_processor to True${COLOR_RESET}"
echo
export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True
fi
RUN_TESTS=${RUN_TESTS:="false"}
CI=${CI:="false"}
# Added to have run-tests on path
export PATH=${PATH}:${AIRFLOW_SOURCES}:/usr/local/go/bin/
mkdir -pv "${AIRFLOW_HOME}/logs/"
# Change the default worker_concurrency for tests
export AIRFLOW__CELERY__WORKER_CONCURRENCY=8
set +e
# shellcheck source=scripts/in_container/configure_environment.sh
. "${IN_CONTAINER_DIR}/configure_environment.sh"
# shellcheck source=scripts/in_container/run_init_script.sh
. "${IN_CONTAINER_DIR}/run_init_script.sh"
"${IN_CONTAINER_DIR}/check_environment.sh"
ENVIRONMENT_EXIT_CODE=$?
set -e
if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then
echo
echo "Error: check_environment returned ${ENVIRONMENT_EXIT_CODE}. Exiting."
echo
exit ${ENVIRONMENT_EXIT_CODE}
fi
mkdir -p /usr/lib/google-cloud-sdk/bin
touch /usr/lib/google-cloud-sdk/bin/gcloud
ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud
if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then
# Set up the ssh-to-localhost plumbing used by the SSH/SFTP provider tests under a
# dedicated directory instead of ~/.ssh: `breeze --forward-credentials` bind-mounts
# the host's ~/.ssh at /root/.ssh and container startup must never modify it.
breeze_ssh_dir="/root/.breeze-ssh"
mkdir -p "${breeze_ssh_dir}"
chmod 700 "${breeze_ssh_dir}"
if [[ ! -f "${breeze_ssh_dir}/id_rsa" ]]; then
ssh-keygen -t rsa -C airflow-breeze-internal-key -m PEM -P '' -f "${breeze_ssh_dir}/id_rsa" \
>"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1
fi
cp "${breeze_ssh_dir}/id_rsa.pub" "${breeze_ssh_dir}/authorized_keys"
chmod 600 "${breeze_ssh_dir}/id_rsa" "${breeze_ssh_dir}/authorized_keys"
echo "AuthorizedKeysFile ${breeze_ssh_dir}/authorized_keys .ssh/authorized_keys" \
> /etc/ssh/sshd_config.d/airflow-breeze.conf
# The heredoc delimiter must not be "EOF": Dockerfile.ci inlines this script inside
# a COPY <<"EOF" heredoc and a bare EOF line would terminate it early.
cat > /etc/ssh/ssh_config.d/airflow-breeze.conf <<SSH_CONFIG
Host localhost 127.0.0.1 ::1
IdentityFile ${breeze_ssh_dir}/id_rsa
UserKnownHostsFile ${breeze_ssh_dir}/known_hosts
StrictHostKeyChecking accept-new
SSH_CONFIG
# Paramiko-based hooks do not read /etc/ssh/ssh_config.d but try ssh-agent keys by
# default, so expose the key through an agent. The fixed socket path lets shells
# entered via entrypoint_exec.sh (breeze exec) pick up the same agent.
rm -f "${breeze_ssh_dir}/agent.sock"
eval "$(ssh-agent -s -a "${breeze_ssh_dir}/agent.sock")" >/dev/null
ssh-add "${breeze_ssh_dir}/id_rsa" >>"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1
sudo service ssh restart >/dev/null 2>&1
# Sometimes the server is not quick enough to load the keys! OpenSSH 10 also prints a
# "# localhost:22 SSH-2.0-..." comment line per connection to stdout, so count only the keys.
while [[ $(ssh-keyscan -H localhost 2>/dev/null | grep -cv '^#') != "3" ]] ; do
echo "Not all keys yet loaded by the server"
sleep 0.05
done
ssh-keyscan -H localhost > "${breeze_ssh_dir}/known_hosts" 2>/dev/null
fi
if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then
echo
echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}"
echo
# Define LocalStack AWS configuration
declare -A localstack_config=(
["AWS_ENDPOINT_URL"]="http://localstack:4566"
["AWS_ACCESS_KEY_ID"]="test"
["AWS_SECRET_ACCESS_KEY"]="test"
["AWS_DEFAULT_REGION"]="us-east-1"
)
# Export each configuration variable and log it
for key in "${!localstack_config[@]}"; do
export "$key"="${localstack_config[$key]}"
echo " * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}"
done
echo
fi
cd "${AIRFLOW_SOURCES}"
# Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags
# in the UI when testing in Breeze. This might be solved differently in the future
if [[ -d /opt/airflow/providers/standard/tests ]]; then
export PYTHONPATH=${PYTHONPATH=}:/opt/airflow/providers/standard/tests
fi
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
if [[ ${BREEZE_DEBUG_CELERY_WORKER=} == "true" ]]; then
export AIRFLOW__CELERY__POOL=${AIRFLOW__CELERY__POOL:-solo}
fi
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
if [[ ${SKIP_ASSETS_COMPILATION:="false"} == "false" ]]; then
wait_for_asset_compilation
fi
if [[ ${TERMINAL_MULTIPLEXER:="mprocs"} == "mprocs" ]]; then
# shellcheck source=scripts/in_container/bin/run_mprocs
exec run_mprocs
else
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
fi
fi
}
function handle_mount_sources() {
if [[ ${MOUNT_SOURCES=} == "remove" ]]; then
echo
echo "${COLOR_BLUE}Mounted sources are removed, cleaning up mounted dist-info files${COLOR_RESET}"
echo
rm -rf /usr/local/lib/python"${PYTHON_MAJOR_MINOR_VERSION}"/site-packages/apache_airflow*.dist-info/
fi
}
function determine_airflow_to_use() {
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
if [[ "${USE_AIRFLOW_VERSION}" == "" && "${USE_DISTRIBUTIONS_FROM_DIST=}" != "true" ]]; then
export PYTHONPATH=${AIRFLOW_SOURCES}
echo
echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}"
echo
# Cleanup the logs, tmp when entering the environment
sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
mkdir -p "${AIRFLOW_SOURCES}"/logs/
mkdir -p "${AIRFLOW_SOURCES}"/tmp/
else
if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then
echo
echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}"
echo
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | grep -v "@" | \
xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS}
# Now install rich ad click first to use the installation script
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} rich rich-click click \
--constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
fi
echo
echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}"
echo
# Generate constraints from uv.lock and use them to install development dependencies
# via the Python script. --no-cache is needed - otherwise there is possibility of
# overriding temporary environments by multiple parallel processes
# --all-packages: without it the export only covers the root project's dependencies.
# Anything that only a workspace member's dependency group pulls in, such
# as mcp through common.ai's pydantic-ai-slim[mcp], stays unpinned and resolves fresh
# from PyPI instead of following uv.lock.
local constraint_file="/tmp/constraints-from-lock.txt"
uv export --frozen --no-hashes --no-emit-project --no-emit-workspace --no-editable --no-header \
--no-annotate --all-packages > "${constraint_file}" 2>/dev/null || true
uv run --no-cache /opt/airflow/scripts/in_container/install_development_dependencies.py \
--constraint "${constraint_file}"
# Some packages might leave legacy typing module which causes test issues
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} typing || true
echo
echo "${COLOR_BLUE}Installing airflow and providers ${COLOR_RESET}"
echo
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
fi
if [[ "${USE_AIRFLOW_VERSION}" =~ ^2.* ]]; then
# Remove auth manager setting
unset AIRFLOW__CORE__AUTH_MANAGER
fi
if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
# make sure old variable is used for older airflow versions
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}"
fi
}
function check_boto_upgrade() {
if [[ ${UPGRADE_BOTO=} != "true" ]]; then
return
fi
echo
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
echo
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
# Urllib 2.6.0 broke kubernetes client by removing getheaders() (restored in 2.6.1+).
# Exclude exactly 2.6.0; any other version is fine.
# Tracked in https://github.com/kubernetes-client/python/issues/2477
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "boto3" "botocore" "urllib3!=2.6.0"
}
function check_upgrade_sqlalchemy() {
if [[ "${UPGRADE_SQLALCHEMY=}" != "true" ]]; then
return
fi
echo
echo "${COLOR_BLUE}Upgrading sqlalchemy to the latest version to run tests with it${COLOR_RESET}"
echo
uv sync --all-packages --resolution highest --no-python-downloads --no-managed-python
}
function check_downgrade_sqlalchemy() {
if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then
return
fi
local min_sqlalchemy_version
min_sqlalchemy_version=$(grep "sqlalchemy\[asyncio\]>=" airflow-core/pyproject.toml | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs)
echo
echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}"
echo
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy[asyncio]==${min_sqlalchemy_version}"
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
pip check
}
function check_downgrade_pendulum() {
if [[ ${DOWNGRADE_PENDULUM=} != "true" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.12" ]]; then
return
fi
local min_pendulum_version
min_pendulum_version=$(grep "pendulum>=" airflow-core/pyproject.toml | head -1 | sed "s/.*>=\([0-9\.]*\).*/\1/" | xargs)
echo
echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}"
echo
# shellcheck disable=SC2086
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}"
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
# We use pip check here to make sure that whatever `uv` installs, is also "correct" according to `pip`
pip check
}
function check_run_tests() {
if [[ ${RUN_TESTS=} != "true" ]]; then
return
fi
if [[ ${TEST_GROUP:=""} == "system" ]]; then
exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}"
else
exec "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${@}"
fi
}
function reinstall_shared_distributions() {
# The shared distributions under shared/<name>/ are workspace members that are not
# transitively required by airflow-core or any provider, so the lowest-direct
# `uv sync` above wipes them out from the environment. Re-install them with
# --no-deps so that the `airflow_shared.*` namespace used by devel-common test
# utils resolves at test collection time without disturbing the lowest-direct
# resolution that was just applied.
echo
echo "${COLOR_BLUE}Re-installing shared distributions (airflow_shared.*) after uv sync${COLOR_RESET}"
echo
# shellcheck disable=SC2046
uv pip install --no-deps $(ls -d /opt/airflow/shared/*/)
}
PROVIDERS_NEEDING_PRE_EXTRAS_INSTALL=()
function run_pre_extras_install_if_registered() {
local provider_id="${1}"
local registered_provider
for registered_provider in "${PROVIDERS_NEEDING_PRE_EXTRAS_INSTALL[@]}"; do
if [[ "${registered_provider}" == "${provider_id}" ]]; then
echo
echo "${COLOR_BLUE}Running pre-extras install manifest for ${provider_id}${COLOR_RESET}"
echo
local env_file
env_file=$(mktemp)
if ! python "${AIRFLOW_SOURCES}/scripts/in_container/run_pre_extras_install.py" \
"${provider_id}" --emit-env-to "${env_file}"; then
rm -f "${env_file}"
echo "${COLOR_RED}Pre-extras install failed for ${provider_id}${COLOR_RESET}"
exit 1
fi
if [[ -s "${env_file}" ]]; then
# shellcheck disable=SC1090
source "${env_file}"
fi
rm -f "${env_file}"
return
fi
done
}
function check_force_lowest_dependencies() {
if [[ ${FORCE_LOWEST_DEPENDENCIES=} != "true" ]]; then
return
fi
if [[ ${TEST_TYPE=} =~ Providers\[.*\] ]]; then
local provider_id
# shellcheck disable=SC2001
provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/')
# A split provider's test type names one part of its tests: Providers[<id>:<part>]
provider_id=${provider_id%%:*}
echo
echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}"
echo
if ! /opt/airflow/scripts/in_container/is_provider_excluded.py "${provider_id}"; then
echo
echo "S${COLOR_YELLOW}Skipping ${provider_id} provider check on Python ${PYTHON_MAJOR_MINOR_VERSION}!${COLOR_RESET}"
echo
exit 0
fi
cd "${AIRFLOW_SOURCES}/providers/${provider_id/.//}" || exit 1
run_pre_extras_install_if_registered "${provider_id}"
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
local sync_successful="false"
for attempt in 1 2 3; do
echo "Attempt ${attempt} of syncing to lowest dependencies"
set -x
if UV_LOCK_TIMEOUT=200 uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
--no-python-downloads --no-managed-python; then
set +x
sync_successful="true"
break
fi
set +x
echo "Sleeping 30s"
sleep 30
echo "Attempt ${attempt} failed. Retrying..."
done
if [[ "${sync_successful}" != "true" ]]; then
echo "${COLOR_RED}Failed to sync lowest dependencies after 3 attempts.${COLOR_RESET}"
exit 1
fi
else
echo
echo "${COLOR_BLUE}Forcing dependencies to lowest versions for Airflow.${COLOR_RESET}"
echo
cd "${AIRFLOW_SOURCES}/airflow-core"
# --no-binary is needed in order to avoid libxml and xmlsec using different version of libxml2
# (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
--no-python-downloads --no-managed-python
fi
reinstall_shared_distributions
}
function check_airflow_python_client_installation() {
if [[ ${INSTALL_AIRFLOW_PYTHON_CLIENT=} != "true" ]]; then
return
fi
python "${IN_CONTAINER_DIR}/install_airflow_python_client.py"
}
function initialize_db() {
# If we are going to start the api server OR we are a system test (which may or may not start the api server,
# depending on the Airflow version being used to run the tests), then migrate the DB.
if [[ ${START_API_SERVER_WITH_EXAMPLES=} == "true" || ${TEST_GROUP:=""} == "system" ]]; then
echo
echo "${COLOR_BLUE}Initializing database${COLOR_RESET}"
echo
airflow db migrate
echo
echo "${COLOR_BLUE}Database initialized${COLOR_RESET}"
fi
}
function start_api_server_with_examples(){
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
# Do not start the api server if either START_API_SERVER_WITH_EXAMPLES is false or the TEST_GROUP env var is not
# equal to "system".
if [[ ${START_API_SERVER_WITH_EXAMPLES=} != "true" && ${TEST_GROUP:=""} != "system" ]]; then
return
fi
# If the use Airflow version is set and it is <= 3.0.0 (which does not have the API server anyway) also return
if [[ ${USE_AIRFLOW_VERSION} != "" && ${USE_AIRFLOW_VERSION} < "3.0.0" ]]; then
return
fi
export AIRFLOW__CORE__LOAD_EXAMPLES=True
export AIRFLOW__API__EXPOSE_CONFIG=True
airflow dags reserialize
echo "Example dags parsing finished"
if airflow config get-value core auth_manager | grep -q "FabAuthManager"; then
echo "Create admin user"
airflow users create -u admin -p admin -f Thor -l Administrator -r Admin -e admin@email.domain || true
echo "Admin user created"
else
echo "Skipping user creation as auth manager different from Fab is used"
fi
echo
echo "${COLOR_BLUE}Starting airflow api server${COLOR_RESET}"
echo
if [[ ${START_API_SERVER_DAEMON:-"true"} == "false" ]]; then
airflow api-server --port 8080 &
else
airflow api-server --port 8080 --daemon
fi
echo
echo "${COLOR_BLUE}Waiting for api-server to start${COLOR_RESET}"
echo
check_service_connection "Airflow api-server" "run_nc localhost 8080" 100
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
echo
echo "${COLOR_RED}Api server did not start properly${COLOR_RESET}"
echo
exit ${EXIT_CODE}
fi
echo
echo "${COLOR_BLUE}Airflow api-server started${COLOR_RESET}"
}
handle_mount_sources
determine_airflow_to_use
environment_initialization
check_boto_upgrade
check_upgrade_sqlalchemy
check_downgrade_sqlalchemy
check_downgrade_pendulum
check_force_lowest_dependencies
check_airflow_python_client_installation
initialize_db
start_api_server_with_examples
check_run_tests "${@}"
exec /bin/bash "${@}"
EOF
# The content below is automatically copied from scripts/docker/entrypoint_exec.sh
COPY <<"EOF" /entrypoint_exec.sh
#!/usr/bin/env bash
. /opt/airflow/scripts/in_container/_in_container_script_init.sh
. /opt/airflow/scripts/in_container/configure_environment.sh
. /opt/airflow/scripts/in_container/run_init_script.sh
if [[ -S /root/.breeze-ssh/agent.sock ]]; then
export SSH_AUTH_SOCK="/root/.breeze-ssh/agent.sock"
fi
exec /bin/bash "${@}"
EOF
# Extract manifests separately so COPY below depends on their contents, not all source files.
FROM ${BASE_IMAGE} as dependency-manifests
WORKDIR /source
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
RUN --mount=type=bind,target=/source \
manifest_list="$(mktemp)" && \
trap 'rm -f "${manifest_list}"' EXIT && \
find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \
mkdir -p /dependency-manifests && \
while IFS= read -r -d '' manifest; do \
relative_path="${manifest#./}"; \
mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \
cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \
done < "${manifest_list}" && \
if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi
FROM ${BASE_IMAGE} as main
# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
# xtrace - to show commands executed)
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
ARG BASE_IMAGE
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
# By increasing this number we can do force build of all dependencies.
# NOTE! When you want to make sure dependencies are installed from scratch in your PR after removing
# some dependencies, you also need to set "disable image cache" in your PR to make sure the image is
# not built using the "main" version of those dependencies.
ARG DEPENDENCIES_EPOCH_NUMBER="15"
# Make sure noninteractive debian install is used and language variables set
ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
INSTALL_MYSQL_CLIENT="true" \
INSTALL_MSSQL_CLIENT="true" \
INSTALL_POSTGRES_CLIENT="true" \
PIP_CACHE_DIR=/root/.cache/pip \
UV_CACHE_DIR=/root/.cache/uv
# GCC 14 in trixie turns incompatible-pointer-types, int-conversion and implicit function declarations
# into errors, and the hardened Python passes -Werror=implicit-function-declaration on to C extensions.
# The lowest-dependency tests build old sdists (for example xmlsec 1.3.14 or ibm-db 3.2.0) that only
# compiled because those were warnings - extensions append CFLAGS after Python's own flags, so this
# turns them back into warnings for every build in the CI image.
ENV CFLAGS="-Wno-error=incompatible-pointer-types -Wno-error=int-conversion -Wno-error=implicit-function-declaration"
RUN echo "Base image version: ${BASE_IMAGE}"
ARG DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_ENV_VARS=""
# No libenchant-2-dev: pyenchant finds libenchant-2-2 through the ldconfig cache, and on trixie the
# -dev package's build dependencies pull in a second Python from the hardened images' repository.
ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git gdb graphviz krb5-user \
less libenchant-2-2 libgeos-dev libpq-dev net-tools \
netcat-openbsd openssh-server postgresql-client rsync tmux \
unzip vim xxd"
ARG ADDITIONAL_DEV_APT_ENV=""
ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
# Empty takes the Python of BASE_IMAGE, breeze passes the patchlevel of the image it builds on.
ARG AIRFLOW_PYTHON_VERSION=""
ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION}
ENV GOLANG_MAJOR_MINOR_VERSION="1.27.3"
ENV TEMURIN_VERSION="11"
ENV NODEJS_VERSION="22.23.1"
# keep this in sync with package.json files
ENV PNPM_VERSION="10.28.1"
ENV RUSTUP_HOME="/usr/local/rustup"
ENV CARGO_HOME="/usr/local/cargo"
ENV PATH="${CARGO_HOME}/bin:${PATH}"
COPY --from=scripts install_os_dependencies.sh /scripts/docker/
RUN bash /scripts/docker/install_os_dependencies.sh ci
# In case system python is installed, setting LD_LIBRARY_PATH prevents any case the system python
# libraries will be accidentally used before the library installed from sources (which is newer and
# python interpreter might break if accidentally the old system libraries are used.
ENV LD_LIBRARY_PATH="/usr/python/lib"
COPY --from=scripts common.sh /scripts/docker/
# Only copy mysql/mssql installation scripts for now - so that changing the other
# scripts which are needed much later will not invalidate the docker layer here.
COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/
COPY scripts/docker/keys/ /scripts/docker/keys/
ARG HOME=/root
ARG AIRFLOW_IMAGE_TYPE="ci"
ARG AIRFLOW_HOME=/root/airflow
ARG AIRFLOW_SOURCES=/opt/airflow
ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb"
ENV HOME=${HOME} \
AIRFLOW_IMAGE_TYPE=${AIRFLOW_IMAGE_TYPE} \
AIRFLOW_HOME=${AIRFLOW_HOME} \
AIRFLOW_SOURCES=${AIRFLOW_SOURCES} \
INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE}
# We run scripts with bash here to make sure we can execute the scripts. Changing to +x might have an
# unexpected result - the cache for Dockerfiles might get invalidated in case the host system
# had different umask set and group x bit was not set. In Azure the bit might be not set at all.
# That also protects against AUFS Docker backend problem where changing the executable bit required sync
RUN bash /scripts/docker/install_mysql.sh prod \
&& bash /scripts/docker/install_mysql.sh dev \
&& bash /scripts/docker/install_mssql.sh dev \
&& bash /scripts/docker/install_postgres.sh dev \
# The user is added to allow ssh debugging (you can connect with airflow/airflow over ssh)
&& adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
--quiet "airflow" --home "/home/airflow" \
&& echo -e "airflow\nairflow" | passwd airflow 2>&1 \
&& echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
&& chmod 0440 /etc/sudoers.d/airflow
# Install Helm
ARG HELM_VERSION="v3.19.0"
RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
&& PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" ) \
&& HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-${PLATFORM}.tar.gz" \
&& curl --silent --location --fail --retry 3 --retry-delay 5 -o /tmp/helm.tar.gz "${HELM_URL}" \
&& tar -xzf /tmp/helm.tar.gz -O "${SYSTEM}-${PLATFORM}/helm" > /usr/local/bin/helm \
&& rm -f /tmp/helm.tar.gz \
&& chmod +x /usr/local/bin/helm
# Install mprocs - a modern process manager for managing multiple Airflow components
ARG MPROCS_VERSION="0.10.3"
RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
&& PLATFORM="$(uname -m)" \
&& MPROCS_URL="https://github.com/pvolok/dekit/releases/download/v${MPROCS_VERSION}/dekit-${PLATFORM}-unknown-${SYSTEM}-musl.tar.gz" \
&& echo "Downloading mprocs from ${MPROCS_URL}" \
&& curl --silent --location --fail --retry 3 --retry-delay 5 -o /tmp/mprocs.tar.gz "${MPROCS_URL}" \
&& tar -xzf /tmp/mprocs.tar.gz -O dekit > /usr/local/bin/mprocs \
&& rm -f /tmp/mprocs.tar.gz \
&& chmod +x /usr/local/bin/mprocs
WORKDIR ${AIRFLOW_SOURCES}
RUN mkdir -pv ${AIRFLOW_HOME} && \
mkdir -pv ${AIRFLOW_HOME}/dags && \
mkdir -pv ${AIRFLOW_HOME}/logs
ARG AIRFLOW_REPO=apache/airflow
ARG AIRFLOW_BRANCH=main
# Airflow Extras installed
ARG AIRFLOW_EXTRAS="all"
ARG ADDITIONAL_AIRFLOW_EXTRAS=""
# Allows to override constraints source
ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow"
ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers"
ARG AIRFLOW_CONSTRAINTS_REFERENCE=""
ARG AIRFLOW_CONSTRAINTS_LOCATION=""
ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
# By default fallback to installation without constraints because in CI image it should always be tried
ARG AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION="true"
# By changing the epoch we can force reinstalling Airflow and pip all dependencies
# It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
ARG AIRFLOW_CI_BUILD_EPOCH="10"
# Setup PIP
# By default PIP has progress bar but you can disable it.
ARG PIP_PROGRESS_BAR="on"
# Optimizing installation of Cassandra driver (in case there are no prebuilt wheels which is the
# case as of 20.04.2021 with Python 3.9
# Speeds up building the image - cassandra driver without CYTHON saves around 10 minutes
ARG CASS_DRIVER_NO_CYTHON="1"
# Build cassandra driver on multiple CPUs
ARG CASS_DRIVER_BUILD_CONCURRENCY="8"
# This value should be set by the CI image build system to the current timestamp
ARG AIRFLOW_VERSION=""
# Additional PIP flags passed to all pip install commands except reinstalling pip itself
ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
ARG AIRFLOW_USE_UV="true"
ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \
AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \
AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \
AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \
AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION=${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} \
DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \
AIRFLOW_VERSION=${AIRFLOW_VERSION} \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \
UV_SYSTEM_PYTHON="true" \
UV_PROJECT_ENVIRONMENT="/usr/local" \
INSTALL_MYSQL_CLIENT="true" \
INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \
INSTALL_MSSQL_CLIENT="true" \
INSTALL_POSTGRES_CLIENT="true" \
AIRFLOW_INSTALLATION_METHOD="." \
AIRFLOW_VERSION_SPECIFICATION="" \
PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \
INCLUDE_PRE_RELEASE="true" \
CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \
CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON}
RUN echo "Airflow version: ${AIRFLOW_VERSION}"
# Copy all scripts required for installation - changing any of those should lead to
# rebuilding from here
COPY --from=scripts common.sh install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/
# We are first creating a venv where all python packages and .so binaries needed by those are
# installed.
# Here we fix the versions so all subsequent commands will use the versions
# from the sources
# You can swap comments between those two args to test pip from the main version
# When you attempt to test if the version of `pip` from specified branch works for our builds
# Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
ARG AIRFLOW_PIP_VERSION=26.2.2
# ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
ARG AIRFLOW_UV_VERSION=0.13.6
ARG AIRFLOW_PREK_VERSION="0.5.7"
# UV_LINK_MODE=copy is needed since we are using cache mounted from the host
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
UV_LINK_MODE=copy \
AIRFLOW_PREK_VERSION=${AIRFLOW_PREK_VERSION}
# The PATH is needed for python to find installed and cargo to build the wheels
ENV PATH="/usr/python/bin:/root/.local/bin:${PATH}"
# Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from
# an incorrect architecture.
ARG TARGETARCH
# Value to be able to easily change cache id and therefore use a bare new cache
ARG DEPENDENCY_CACHE_EPOCH="2"
# Install useful command line tools in their own virtualenv so that they do not clash with
# dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed
# in the version specified above
RUN bash /scripts/docker/install_packaging_tools.sh
COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/
COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/
ARG UPGRADE_RANDOM_INDICATOR_STRING=""
# Only preinstall locked third-party dependencies, with the same uv sync as the full installation.
# A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still
# installs workspace packages from the current checkout and retains its resolution fallback. Any other
# failure would fail the full sync as well, so it fails the build here.
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \
if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \
INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \
fi
# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
# if you want to add something that is missing and you expect to see it in the image you can
# add it with ! in .dockerignore next to the airflow, test etc. directories there
COPY . ${AIRFLOW_SOURCES}/
ARG VERSION_SUFFIX=""
ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \
VERSION_SUFFIX=${VERSION_SUFFIX}
# The goal of this line is to install the dependencies from the most current pyproject.toml from sources
# This will be usually incremental small set of packages in CI optimized build, so it will be very fast
# In non-CI optimized build this will install all dependencies before installing sources.
# Usually we will install versions based on the dependencies in pyproject.toml and upgraded only if needed.
# But in cron job we will install latest versions matching pyproject.toml to see if there is no breaking change
# and push the constraints if everything is successful
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ bash /scripts/docker/install_airflow_when_building_images.sh
COPY --from=scripts install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/
ARG ADDITIONAL_PYTHON_DEPS=""
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS}
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \
bash /scripts/docker/install_packaging_tools.sh; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
bash /scripts/docker/install_additional_dependencies.sh; \
fi
COPY --from=scripts entrypoint_ci.sh /entrypoint
COPY --from=scripts entrypoint_exec.sh /entrypoint-exec
RUN chmod a+x /entrypoint /entrypoint-exec
# Install autocomplete for airflow and kubectl
# hadolint ignore=SC2028
RUN if command -v airflow; then \
register-python-argcomplete airflow >> ~/.bashrc ; \
fi; \
echo "source /etc/bash_completion" >> ~/.bashrc ; \
echo 'export PS1="\[\033[1;36m\][Breeze:\$(python --version 2>&1 | cut -d\" \" -f2)]\[\033[0m\] \[\033[1;32m\]\u@\h\[\033[0m\]:\[\033[1;34m\]\w\[\033[0m\]\$ "' >> ~/.bashrc
WORKDIR ${AIRFLOW_SOURCES}
ARG BUILD_ID
ARG COMMIT_SHA
ARG AIRFLOW_IMAGE_DATE_CREATED
ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${PATH}" \
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" \
BUILD_ID=${BUILD_ID} \
COMMIT_SHA=${COMMIT_SHA} \
# When we enter the image, the /root/.cache is not mounted from temporary mount cache.
# We do not want to share the cache from host to avoid all kinds of problems where cache
# is different with different platforms / python versions. We want to have a clean cache
# in the image - and in this case /root/.cache is on the same filesystem as the installed packages.
# so we can go back to the default link mode being hardlink.
UV_LINK_MODE=hardlink \
MYPY_FORCE_COLOR="true"
# Link dumb-init for backwards compatibility (so that older images also work)
RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
EXPOSE 8080
LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.module="airflow" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow-ci" \
org.apache.airflow.version="${AIRFLOW_VERSION}" \
org.apache.airflow.python.version="${AIRFLOW_PYTHON_VERSION}" \
org.apache.airflow.uid="0" \
org.apache.airflow.gid="0" \
org.apache.airflow.build-id="${BUILD_ID}" \
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
org.opencontainers.image.source="${AIRFLOW_IMAGE_REPOSITORY}" \
org.opencontainers.image.created="${AIRFLOW_IMAGE_DATE_CREATED}" \
org.opencontainers.image.authors="dev@airflow.apache.org" \
org.opencontainers.image.url="https://airflow.apache.org" \
org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \
org.opencontainers.image.source="https://github.com/apache/airflow" \
org.opencontainers.image.version="${AIRFLOW_VERSION}" \
org.opencontainers.image.revision="${COMMIT_SHA}" \
org.opencontainers.image.vendor="Apache Software Foundation" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.ref.name="airflow-ci-image" \
org.opencontainers.image.title="Continuous Integration Airflow Image" \
org.opencontainers.image.description="Installed Apache Airflow with Continuous Integration dependencies"
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
CMD []
Reference in new issue View Git Blame Copy Permalink