diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 9fe5b551096..d7c780e06e0 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -22,6 +22,11 @@ inputs: python-version: description: 'Python version to use' default: "3.11" + free-space: + description: >- + Whether to free disk space on the runner. It needs swap control and docker, which the ubuntu-slim + container runner does not have, so jobs on ubuntu-slim pass "false". + default: "true" outputs: host-python-version: description: Python version used in host @@ -55,6 +60,7 @@ runs: env: PYTHON_VERSION: "${{ inputs.python-version }}" - name: "Free space" + if: inputs.free-space == 'true' shell: bash run: breeze ci free-space env: diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index fb88f09e343..8af50f522b8 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -263,8 +263,9 @@ jobs: if: >- needs.gate.outputs.project-ci == 'true' && needs.check-run-needed.outputs.run-needed == 'true' - # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners - runs-on: ["ubuntu-22.04"] + # At build-info stage we do not yet have outputs so the runs-on is hard-coded. ubuntu-slim is enough: + # it only installs breeze and runs selective checks, which need neither docker nor freeing disk space. + runs-on: ["ubuntu-slim"] outputs: all-python-versions-list-as-string: >- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} @@ -378,9 +379,6 @@ jobs: upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -403,6 +401,8 @@ jobs: || echo "::warning::Could not fetch ${PUSH_BASE_COMMIT}, all tests will run" - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" id: breeze - name: "Save github context to file" # The github context can be very large for PRs that touch many files (e.g. Dependabot diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 7b88dea02a8..09b0967b490 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -144,8 +144,9 @@ jobs: name: "Build info" needs: [check-run-needed] if: needs.check-run-needed.outputs.run-needed == 'true' - # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners - runs-on: ["ubuntu-22.04"] + # At build-info stage we do not yet have outputs so the runs-on is hard-coded. ubuntu-slim is enough: + # it only installs breeze and runs selective checks, which need neither docker nor freeing disk space. + runs-on: ["ubuntu-slim"] outputs: all-python-versions-list-as-string: >- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} @@ -259,9 +260,6 @@ jobs: upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -284,6 +282,8 @@ jobs: || echo "::warning::Could not fetch ${PUSH_BASE_COMMIT}, all tests will run" - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" id: breeze - name: "Save github context to file" # The github context can be very large for PRs that touch many files (e.g. Dependabot diff --git a/.github/workflows/mirror-hardened-base-images.yml b/.github/workflows/mirror-hardened-base-images.yml index b7eac54c318..cfba2ab849b 100644 --- a/.github/workflows/mirror-hardened-base-images.yml +++ b/.github/workflows/mirror-hardened-base-images.yml @@ -27,31 +27,80 @@ name: Mirror hardened base images # # `docker buildx imagetools create` copies the multi-platform manifest between registries without # pulling the layers into the runner, so a refresh of all Python versions is cheap. +# +# The builds do not use these moving tags: they are pinned to digests that only "breeze ci upgrade" +# moves, in a PR that has to pass the full CI. Docker rebuilds the images in place and moves the +# dhi.io apt repository on together with them, so a pin left behind soon stops matching what apt +# installs. A run that moves a tag past its pin therefore triggers that upgrade check, and the pins +# follow Docker's rebuilds within hours. on: # yamllint disable-line rule:truthy workflow_dispatch: + inputs: + trigger-upgrade-check: + description: >- + Run the main upgrade check when a tag moves past its pin. "breeze ci upgrade" for main turns + it off, as it is the upgrade that pins the images. + type: boolean + default: true schedule: - # Docker rebuilds the hardened images as CVEs are fixed, keeping the same tags. Re-running - # weekly is what picks those rebuilds up. - - cron: '27 4 * * 1' + - cron: '27 */6 * * *' permissions: contents: read concurrency: group: mirror-hardened-base-images cancel-in-progress: false jobs: + # Picks the runner the way CI does: RunsOn in apache/airflow, public runners when the repository + # variable USE_PUBLIC_RUNNERS is "true". + build-info: + name: "Build info" + runs-on: ["ubuntu-slim"] + timeout-minutes: 10 + if: github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' + permissions: + contents: read + outputs: + runner-type: ${{ steps.selective-checks.outputs.runner-type }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + free-space: "false" + - name: "Save github context to file" + # Same as in ci-amd.yml: a scheduled or dispatched run has no attacker-controlled context, and + # the single-quoted heredoc disables every shell expansion of the JSON anyway. + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + PLATFORM: "linux/amd64" + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} mirror: name: "Mirror hardened base images" - runs-on: ["ubuntu-22.04"] + needs: [build-info] + # Copies manifests registry-to-registry without pulling layers, so the smallest class is enough. + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '2+4', '8+16') }} + timeout-minutes: 30 # Never runs on forks - they have neither the credentials nor the package to push to. Only main - # and the v3-*-test release branches may push: the mirror is the default BASE_IMAGE of every - # build, so a manual dispatch from any other branch must not be able to replace it. - if: >- - github.repository == 'apache/airflow' && ( - github.ref == 'refs/heads/main' || ( - startsWith(github.ref, 'refs/heads/v3-') && endsWith(github.ref, '-test') - ) - ) + # pushes: the mirror is the default BASE_IMAGE of every build, so a manual dispatch from any other + # branch must not be able to replace it. Main mirrors every Python version an active release + # branch still builds on, so the release branches need no runs of their own. + if: github.repository == 'apache/airflow' && github.ref == 'refs/heads/main' permissions: + actions: write contents: read packages: write steps: @@ -77,4 +126,15 @@ jobs: ACTOR: ${{ github.actor }} run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${ACTOR}" --password-stdin - name: "Mirror base images" + id: mirror run: breeze release-management mirror-base-images + # Only main's pins: release branches move theirs on their own upgrade schedule. Also after a + # failed mirror step: a tag it moved before failing stays moved, so a re-run would not see it + # move again. + - name: "Trigger the upgrade check to move the pinned base images" + if: >- + !cancelled() && steps.mirror.outputs.moved-past-pins == 'true' && + (github.event_name == 'schedule' || inputs.trigger-upgrade-check) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run scheduled-upgrade-check-main.yml --repo "${GITHUB_REPOSITORY}" --ref main diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 4deb0b9132b..8cc9f6829fd 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -75,7 +75,7 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - runs-on: ["ubuntu-24.04"] + runs-on: ["ubuntu-slim"] env: VERBOSE: true REF: ${{ inputs.ref }} diff --git a/.github/workflows/refresh-image-registry-cache.yml b/.github/workflows/refresh-image-registry-cache.yml index e133222cf2d..2eb1821f474 100644 --- a/.github/workflows/refresh-image-registry-cache.yml +++ b/.github/workflows/refresh-image-registry-cache.yml @@ -65,7 +65,7 @@ jobs: build-info: name: "Build info" # No outputs yet at this stage, so the runner has to be hard-coded to a public one. - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-slim"] outputs: default-branch: ${{ steps.selective-checks.outputs.default-branch }} default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} @@ -75,15 +75,14 @@ jobs: python-versions: ${{ steps.selective-checks.outputs.python-versions }} use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" - name: Selective checks id: selective-checks env: diff --git a/.github/workflows/release-constraints.yml b/.github/workflows/release-constraints.yml index f1f1437c58f..cceb80cd5ee 100644 --- a/.github/workflows/release-constraints.yml +++ b/.github/workflows/release-constraints.yml @@ -52,7 +52,7 @@ jobs: build-info: timeout-minutes: 10 name: "Build info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-slim"] if: contains(fromJSON('[ "ashb", "eladkal", @@ -74,9 +74,6 @@ jobs: target-branch: ${{ steps.stage.outputs.target-branch }} allow-pre-releases: ${{ steps.stage.outputs.allow-pre-releases }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ inputs.ref }}" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -85,6 +82,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" id: breeze - name: "Save github context to file" # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index fb81f83d8ac..dad4ac52f7e 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -44,7 +44,7 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - runs-on: ["ubuntu-24.04"] + runs-on: ["ubuntu-slim"] outputs: pythonVersions: ${{ steps.determine-python-versions.outputs.python-versions }} platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} @@ -81,17 +81,14 @@ jobs: echo "Airflow version: '${AIRFLOW_VERSION}'" echo "AMD only: '${AMD_ONLY}'" echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'" - - name: "Cleanup repo" - shell: bash - run: > - sudo rm -rf ${GITHUB_WORKSPACE}/* - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" - name: "Save github context to file" # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the # github context to a file, and the single-quoted heredoc makes the zizmor diff --git a/.github/workflows/scheduled-legacy-prod-image-canary.yml b/.github/workflows/scheduled-legacy-prod-image-canary.yml index 1f069498daa..db8b8a154eb 100644 --- a/.github/workflows/scheduled-legacy-prod-image-canary.yml +++ b/.github/workflows/scheduled-legacy-prod-image-canary.yml @@ -36,7 +36,7 @@ concurrency: jobs: build-info: name: "Build info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-slim"] if: github.repository == 'apache/airflow' || github.event_name == 'workflow_dispatch' timeout-minutes: 15 outputs: @@ -72,9 +72,6 @@ jobs: run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -82,6 +79,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" - name: "Save github context to file" # Same as in ci-amd.yml: a scheduled or dispatched run has no attacker-controlled context, and # the single-quoted heredoc disables every shell expansion of the JSON anyway. diff --git a/.github/workflows/scheduled-upgrade-check-main.yml b/.github/workflows/scheduled-upgrade-check-main.yml index 1c828f86b03..ee0ab72a15a 100644 --- a/.github/workflows/scheduled-upgrade-check-main.yml +++ b/.github/workflows/scheduled-upgrade-check-main.yml @@ -25,6 +25,8 @@ on: # yamllint disable-line rule:truthy permissions: contents: write pull-requests: write + # Only the job of upgrade-check.yml that dispatches the "Mirror hardened base images" workflow gets it. + actions: write jobs: upgrade-main: name: "[main] Upgrade" diff --git a/.github/workflows/scheduled-upgrade-check-v3-4-test.yml b/.github/workflows/scheduled-upgrade-check-v3-4-test.yml index 604fcb2ed8f..9d51e87cec4 100644 --- a/.github/workflows/scheduled-upgrade-check-v3-4-test.yml +++ b/.github/workflows/scheduled-upgrade-check-v3-4-test.yml @@ -25,6 +25,8 @@ on: # yamllint disable-line rule:truthy permissions: contents: write pull-requests: write + # Only the job of upgrade-check.yml that dispatches the "Mirror hardened base images" workflow gets it. + actions: write jobs: upgrade-v3-4-test: name: "[v3-4-test] Upgrade" diff --git a/.github/workflows/update-constraints-on-push-stable.yml b/.github/workflows/update-constraints-on-push-stable.yml index 47379a8933d..b4904dbed54 100644 --- a/.github/workflows/update-constraints-on-push-stable.yml +++ b/.github/workflows/update-constraints-on-push-stable.yml @@ -35,7 +35,7 @@ env: jobs: build-info: name: "Build info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-slim"] outputs: default-branch: ${{ steps.selective-checks.outputs.default-branch }} default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} @@ -43,9 +43,6 @@ jobs: python-versions: ${{ steps.selective-checks.outputs.python-versions }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -58,6 +55,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" id: breeze - name: "Save github context to file" # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the diff --git a/.github/workflows/update-constraints-on-push.yml b/.github/workflows/update-constraints-on-push.yml index 4ffe806242f..6dbce8cbac4 100644 --- a/.github/workflows/update-constraints-on-push.yml +++ b/.github/workflows/update-constraints-on-push.yml @@ -73,7 +73,7 @@ env: jobs: build-info: name: "Build info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-slim"] # Automatic uv.lock-push runs are unrestricted; manual (workflow_dispatch) # runs are limited to release managers (same allowlist as the prod image # release workflow) because they push to the protected constraints-* branches. @@ -101,9 +101,6 @@ jobs: python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} constraints-baseline-sha: ${{ steps.constraints-baseline.outputs.sha }} steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - name: "Checkout ${{ inputs.ref || github.ref }} ( ${{ inputs.ref || github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -112,6 +109,8 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze + with: + free-space: "false" id: breeze - name: "Save github context to file" # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the diff --git a/.github/workflows/upgrade-check.yml b/.github/workflows/upgrade-check.yml index 1a5747d3c7b..791fd58d925 100644 --- a/.github/workflows/upgrade-check.yml +++ b/.github/workflows/upgrade-check.yml @@ -30,18 +30,89 @@ on: # yamllint disable-line rule:truthy description: "Slack bot token for notifications" required: true permissions: - contents: write - pull-requests: write + contents: read env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} TARGET_BRANCH: ${{ inputs.target-branch }} jobs: + # Picks the runner of the upgrade the way CI does: RunsOn in apache/airflow, public runners in forks + # or when the repository variable USE_PUBLIC_RUNNERS is "true". + build-info: + name: >- + [${{ inputs.target-branch }}] Build info + runs-on: ["ubuntu-slim"] + timeout-minutes: 10 + permissions: + contents: read + outputs: + runner-type: ${{ steps.selective-checks.outputs.runner-type }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + free-space: "false" + - name: "Save github context to file" + # Same as in ci-amd.yml: a scheduled or dispatched run has no attacker-controlled context, and + # the single-quoted heredoc disables every shell expansion of the JSON anyway. + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: "Selective checks" + id: selective-checks + env: + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + PLATFORM: "linux/amd64" + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + # A job of its own, so the token that may dispatch workflows never reaches the upgrade job, which + # runs the third-party tools and hooks it upgrades. It only dispatches the mirror and waits for it. + mirror-base-images: + timeout-minutes: 30 + name: >- + [${{ inputs.target-branch }}] Mirror hardened base images + runs-on: ["ubuntu-slim"] + permissions: + actions: write + contents: read + steps: + - name: >- + [${{ inputs.target-branch }}] Checkout main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + - name: >- + [${{ inputs.target-branch }}] Install Breeze + uses: ./.github/actions/breeze + with: + free-space: "false" + # The upgrade for main is the one the mirror would otherwise start a second time. + - name: >- + [${{ inputs.target-branch }}] Mirror the base images and wait for the run + env: + TRIGGER_UPGRADE_CHECK_FLAG: >- + ${{ inputs.target-branch == 'main' && '--no-trigger-upgrade-check' || '--trigger-upgrade-check' }} + run: breeze workflow-run mirror-base-images "${TRIGGER_UPGRADE_CHECK_FLAG}" createupgrade-check: timeout-minutes: 45 name: >- [${{ inputs.target-branch }}] Upgrade checks and PR - runs-on: ["ubuntu-22.04"] + # Builds the CI image for the uv.lock refresh and runs every hook, like the CI image builds. + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + needs: [build-info, mirror-base-images] + permissions: + contents: write + pull-requests: write steps: - name: >- [${{ inputs.target-branch }}] Cleanup repo @@ -85,6 +156,7 @@ jobs: --switch-to-base --no-k8s-schema-sync --latest-package-managers + --no-mirror-base-images --answer yes - name: >- [${{ inputs.target-branch }}] Find upgrade PR diff --git a/Dockerfile b/Dockerfile index 55f5878e3ce..7f3b986f4dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,18 +54,25 @@ ARG AIRFLOW_VERSION="3.3.2" # image are based on it, so Python is never compiled during the build. # # The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no -# registry credentials. The upstream image itself requires a "docker login dhi.io". +# registry credentials. The upstream image itself requires a "docker login dhi.io". It is pinned to a +# digest, because the mirror's tags move several times a day: "breeze ci upgrade" moves the pin once +# the full CI has passed on the new image. # # The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the # default, but you can build a FIPS-compliant image by pointing the build at one: -# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.16-debian13-fips-dev" +# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.18-debian13-fips-dev" # # AIRFLOW_IMAGE_FLAVOR="legacy" builds the image the way it was built before the hardened base images: # BASE_IMAGE is then a plain Debian image (for example "debian:bookworm-slim") and Python -# AIRFLOW_PYTHON_VERSION is downloaded from python.org, verified and compiled from sources. +# AIRFLOW_PYTHON_VERSION is downloaded from python.org, verified and compiled from sources. A +# MAJOR.MINOR AIRFLOW_PYTHON_VERSION builds the latest release of that line. +# +# AIRFLOW_PYTHON_VERSION is empty by default: the hardened flavor takes Python from BASE_IMAGE, and the +# build fails if a version passed here does not match it. Breeze passes the Python patchlevel that +# "breeze ci upgrade" last moved, so a new patch release also rebuilds the legacy flavor. ARG AIRFLOW_IMAGE_FLAVOR="hardened" -ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" -ARG AIRFLOW_PYTHON_VERSION="3.13.16" +ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.18-debian13-dev@sha256:9ed60ac490ee5038920b1e69ea9adb256f9f95ca285b65800d2ddab73b6e3a1e" +ARG AIRFLOW_PYTHON_VERSION="" # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds @@ -125,7 +132,7 @@ PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as @@ -133,6 +140,13 @@ if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} # Read before apt runs: a package depending on python3 can pull another Python version in. BASE_PYTHON_MAJOR_MINOR=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') + if [[ -n "${AIRFLOW_PYTHON_VERSION}" && "${AIRFLOW_PYTHON_VERSION}." != "${BASE_PYTHON_MAJOR_MINOR}."* ]]; then + echo + echo "ERROR! AIRFLOW_PYTHON_VERSION is ${AIRFLOW_PYTHON_VERSION}, but the base image has Python ${BASE_PYTHON_MAJOR_MINOR}." + echo "The hardened flavor takes Python from BASE_IMAGE: leave AIRFLOW_PYTHON_VERSION empty or pick a BASE_IMAGE with that Python." + echo + exit 1 + fi elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else @@ -307,6 +321,24 @@ function keep_image_conffiles() { printf '%s\n' force-confdef force-confold > /etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles } +function prefer_hardened_apt_repository() { + # The hardened base images take their packages from dhi.io, but stock Debian's sources stay + # configured beside it and apt installs whichever repository offers the higher version. Debian's + # build of a package then replaces Docker's, and its dependencies no longer match the packages the + # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above the default priority of + # 500 makes apt take Docker's build whenever dhi.io publishes the package, and fall back to Debian + # only when it does not. + # The priority stays below 1000 so the preference never forces a downgrade: apt still favors + # dhi.io's build of a package it publishes, but may take a newer version from another repository + # when a dependency requires it - for example PGDG's libpq5, which postgresql-client-18 needs at + # ">= 18.6" and dhi.io does not carry. Pinning the base image to a digest CI has passed on keeps + # dhi.io and the image consistent, so the renumbering that once needed a forced "downgrade" of the + # image's copy (docker-hardened-images/catalog#689) no longer arises within a build. + # The preference covers every apt run - this build's and those of images extending it. + mkdir -p /etc/apt/preferences.d + printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' > /etc/apt/preferences.d/airflow-prefer-dhi +} + function refresh_ldconfig_after_dpkg() { # The hardened base images ship libc-bin without its dpkg trigger, so installing a shared library # does not refresh the ldconfig cache and ctypes.util.find_library() - which reads that cache - @@ -366,6 +398,7 @@ function align_perl_base_with_perl() { function install_debian_dev_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles + prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update @@ -548,8 +581,39 @@ function install_cosign() { chmod +x /tmp/cosign } +function resolve_python_version() { + if [[ -z "${AIRFLOW_PYTHON_VERSION}" ]]; then + echo + echo "ERROR! The legacy flavor compiles Python from sources, so AIRFLOW_PYTHON_VERSION has to name it:" + echo "MAJOR.MINOR for the latest release of that line, or MAJOR.MINOR.PATCH." + echo + exit 1 + fi + if [[ ! "${AIRFLOW_PYTHON_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then + return + fi + # A release directory appears on python.org with its first pre-release, so take the newest one + # that already holds the final tarball. + local candidate + for candidate in $(wget --tries=3 --waitretry=5 -qO- https://www.python.org/ftp/python/ \ + | grep -oE "href=\"${AIRFLOW_PYTHON_VERSION//./\\.}\.[0-9]+/\"" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r); do + if wget --tries=3 --waitretry=5 -q --spider \ + "https://www.python.org/ftp/python/${candidate}/Python-${candidate}.tar.xz"; then + echo "Python ${AIRFLOW_PYTHON_VERSION} resolved to its latest release ${candidate}" + AIRFLOW_PYTHON_VERSION="${candidate}" + return + fi + done + echo + echo "ERROR! No release of Python ${AIRFLOW_PYTHON_VERSION} found on python.org." + echo + exit 1 +} + function install_python() { # Only the legacy image flavor compiles Python - the hardened flavor gets it from the base image. + resolve_python_version wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" @@ -630,6 +694,7 @@ function install_python() { function install_debian_runtime_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles + prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update @@ -971,6 +1036,12 @@ install_postgres_client() { echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list + # The hardened image's airflow-prefer-dhi pin outranks Debian's default, so apt would otherwise keep + # dhi.io's libpq (e.g. 17.11) and postgresql-client (which needs a matching, newer libpq5) could not + # install. PostgreSQL's own repository is the authority for its packages - rank it above the dhi.io + # preference so apt takes libpq from PGDG. + mkdir -p /etc/apt/preferences.d + printf 'Package: *\nPin: origin "apt.postgresql.org"\nPin-Priority: 1001\n' > /etc/apt/preferences.d/airflow-prefer-pgdg apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/Dockerfile.ci b/Dockerfile.ci index fd9c2012ca9..8ee5032cb2d 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -22,12 +22,14 @@ # image are based on it, so Python is never compiled during the build. # # The default is Airflow's public mirror of the upstream "dhi.io/python" image, so building needs no -# registry credentials. The upstream image itself requires a "docker login dhi.io". +# registry credentials. The upstream image itself requires a "docker login dhi.io". It is pinned to a +# digest, because the mirror's tags move several times a day: "breeze ci upgrade" moves the pin once +# the full CI has passed on the new image. # # The FIPS-validated variants require a paid Docker subscription and cannot be mirrored or made the # default, but you can build a FIPS-compliant image by pointing the build at one: -# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.16-debian13-fips-dev" -ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.16-debian13-dev" +# docker build . --build-arg BASE_IMAGE="dhi.io/python:3.13.18-debian13-fips-dev" +ARG BASE_IMAGE="ghcr.io/apache/airflow/base/python:3.13.18-debian13-dev@sha256:9ed60ac490ee5038920b1e69ea9adb256f9f95ca285b65800d2ddab73b6e3a1e" ############################################################################################## # This is the script image where we keep all inlined bash scripts needed in other segments @@ -61,7 +63,7 @@ PNPM_VERSION=${PNPM_VERSION:-10.28.1} RUSTUP_DEFAULT_TOOLCHAIN=${RUSTUP_DEFAULT_TOOLCHAIN:-stable} RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened} -AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16} +AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-} COSIGN_VERSION=${COSIGN_VERSION:-3.0.5} if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then # The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as @@ -69,6 +71,13 @@ if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')} # Read before apt runs: a package depending on python3 can pull another Python version in. BASE_PYTHON_MAJOR_MINOR=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') + if [[ -n "${AIRFLOW_PYTHON_VERSION}" && "${AIRFLOW_PYTHON_VERSION}." != "${BASE_PYTHON_MAJOR_MINOR}."* ]]; then + echo + echo "ERROR! AIRFLOW_PYTHON_VERSION is ${AIRFLOW_PYTHON_VERSION}, but the base image has Python ${BASE_PYTHON_MAJOR_MINOR}." + echo "The hardened flavor takes Python from BASE_IMAGE: leave AIRFLOW_PYTHON_VERSION empty or pick a BASE_IMAGE with that Python." + echo + exit 1 + fi elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then PYTHON_HOME=${PYTHON_HOME:-/usr/python} else @@ -243,6 +252,24 @@ function keep_image_conffiles() { printf '%s\n' force-confdef force-confold > /etc/dpkg/dpkg.cfg.d/airflow-keep-conffiles } +function prefer_hardened_apt_repository() { + # The hardened base images take their packages from dhi.io, but stock Debian's sources stay + # configured beside it and apt installs whichever repository offers the higher version. Debian's + # build of a package then replaces Docker's, and its dependencies no longer match the packages the + # image ships (docker-hardened-images/catalog#610). Ranking dhi.io above the default priority of + # 500 makes apt take Docker's build whenever dhi.io publishes the package, and fall back to Debian + # only when it does not. + # The priority stays below 1000 so the preference never forces a downgrade: apt still favors + # dhi.io's build of a package it publishes, but may take a newer version from another repository + # when a dependency requires it - for example PGDG's libpq5, which postgresql-client-18 needs at + # ">= 18.6" and dhi.io does not carry. Pinning the base image to a digest CI has passed on keeps + # dhi.io and the image consistent, so the renumbering that once needed a forced "downgrade" of the + # image's copy (docker-hardened-images/catalog#689) no longer arises within a build. + # The preference covers every apt run - this build's and those of images extending it. + mkdir -p /etc/apt/preferences.d + printf 'Package: *\nPin: origin "dhi.io"\nPin-Priority: 999\n' > /etc/apt/preferences.d/airflow-prefer-dhi +} + function refresh_ldconfig_after_dpkg() { # The hardened base images ship libc-bin without its dpkg trigger, so installing a shared library # does not refresh the ldconfig cache and ctypes.util.find_library() - which reads that cache - @@ -302,6 +329,7 @@ function align_perl_base_with_perl() { function install_debian_dev_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles + prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update @@ -484,8 +512,39 @@ function install_cosign() { chmod +x /tmp/cosign } +function resolve_python_version() { + if [[ -z "${AIRFLOW_PYTHON_VERSION}" ]]; then + echo + echo "ERROR! The legacy flavor compiles Python from sources, so AIRFLOW_PYTHON_VERSION has to name it:" + echo "MAJOR.MINOR for the latest release of that line, or MAJOR.MINOR.PATCH." + echo + exit 1 + fi + if [[ ! "${AIRFLOW_PYTHON_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then + return + fi + # A release directory appears on python.org with its first pre-release, so take the newest one + # that already holds the final tarball. + local candidate + for candidate in $(wget --tries=3 --waitretry=5 -qO- https://www.python.org/ftp/python/ \ + | grep -oE "href=\"${AIRFLOW_PYTHON_VERSION//./\\.}\.[0-9]+/\"" \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r); do + if wget --tries=3 --waitretry=5 -q --spider \ + "https://www.python.org/ftp/python/${candidate}/Python-${candidate}.tar.xz"; then + echo "Python ${AIRFLOW_PYTHON_VERSION} resolved to its latest release ${candidate}" + AIRFLOW_PYTHON_VERSION="${candidate}" + return + fi + done + echo + echo "ERROR! No release of Python ${AIRFLOW_PYTHON_VERSION} found on python.org." + echo + exit 1 +} + function install_python() { # Only the legacy image flavor compiles Python - the hardened flavor gets it from the base image. + resolve_python_version wget --tries=3 --waitretry=5 -O python.tar.xz "https://www.python.org/ftp/python/${AIRFLOW_PYTHON_VERSION%%[a-z]*}/Python-${AIRFLOW_PYTHON_VERSION}.tar.xz" local major_minor_version major_minor_version="${AIRFLOW_PYTHON_VERSION%.*}" @@ -566,6 +625,7 @@ function install_python() { function install_debian_runtime_dependencies() { if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then keep_image_conffiles + prefer_hardened_apt_repository refresh_ldconfig_after_dpkg fi apt-get update @@ -907,6 +967,12 @@ install_postgres_client() { echo "deb [arch=amd64,arm64] https://apt.postgresql.org/pub/repos/apt/ $(common::debian_codename)-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list + # The hardened image's airflow-prefer-dhi pin outranks Debian's default, so apt would otherwise keep + # dhi.io's libpq (e.g. 17.11) and postgresql-client (which needs a matching, newer libpq5) could not + # install. PostgreSQL's own repository is the authority for its packages - rank it above the dhi.io + # preference so apt takes libpq from PGDG. + mkdir -p /etc/apt/preferences.d + printf 'Package: *\nPin: origin "apt.postgresql.org"\nPin-Priority: 1001\n' > /etc/apt/preferences.d/airflow-prefer-pgdg apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge @@ -1979,7 +2045,8 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} -ARG AIRFLOW_PYTHON_VERSION="3.13.16" +# Empty takes the Python of BASE_IMAGE, breeze passes the patchlevel of the image it builds on. +ARG AIRFLOW_PYTHON_VERSION="" ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION} ENV GOLANG_MAJOR_MINOR_VERSION="1.27.2" ENV TEMURIN_VERSION="11" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 8b89af8f6eb..506dd7ee8d8 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -129,6 +129,42 @@ introduced have it already. An older release branch only behaves this way once t `ci-amd.yml` / `ci-arm.yml` that introduced it has been backported to it; until then it keeps running the full CI on every push. +### Before cutting a beta or an RC: refresh the base images and the important CI versions + +The images are built on hardened Python base images pinned to digests, and the legacy images compile +a pinned Python patch release. Only the "Upgrade important CI environment" upgrade moves those pins, +so a release ships with the latest base images, Python patch releases and their security fixes only +when the upgrade ran right before it. It is one command, which first mirrors the latest hardened base +images (a workflow run on `main`, as the mirror is shared by all branches) and waits for it. It needs +`gh` credentials that may dispatch `apache/airflow` workflows, and checks that first. + +- **Beta** - before you fast-forward `vX-Y-test` to `main` for the beta (see + [Beta releases](#beta-releases-fast-forwarding-vx-y-test-to-main)), as during the beta phase + `vX-Y-test` takes its changes only from `main`: + + ```shell script + breeze workflow-run upgrade-check --branch main + ``` + + It runs the upgrade check workflow for `main` and waits for it. When anything moved, the workflow + opens (or updates) the `[main] Upgrade important CI environment` PR as a draft. Mark it ready for + review, which starts its CI, and merge it once the CI has passed. + +- **RC** - before you open the PR syncing `vX-Y-test` into `vX-Y-stable`, and before the full-run + check in the next section: + + ```shell script + git switch vX-Y-test + git pull --ff-only upstream vX-Y-test + breeze workflow-run upgrade-check --branch vX-Y-test + ``` + + It runs the upgrade in your checkout and pushes the `[vX-Y-test] CI: Upgrade important CI + environment` commit straight to `vX-Y-test`, the head of the test-to-stable PR, so the PR's CI tests + it. This runs locally because the release branch rules let only committers push to `vX-Y-test` + without CI having passed. It never switches, resets or cleans your checkout: it fails and tells you + what to do when you are not on `vX-Y-test` at the tip of `apache/airflow`, or have local changes. + ### Before cutting an RC: check the full run of the branch HEAD The incremental push runs do not replace the full run. Before you cut an RC (or open the PR syncing @@ -171,9 +207,12 @@ released from `vX-Y-test` only: there is no `vX-Y-stable` branch yet and therefo PR exists only to carry changes from `vX-Y-test` into `vX-Y-stable` for an `rc`/final), and there is no vote. -1. Push the version-bump and release-notes commits to `vX-Y-test` through normal PRs, so the tip of +1. Run the upgrade check for `main` and merge its PR, as described in + [Before cutting a beta or an RC](#before-cutting-a-beta-or-an-rc-refresh-the-base-images-and-the-important-ci-versions), + then fast-forward `vX-Y-test` to `main`. +2. Push the version-bump and release-notes commits to `vX-Y-test` through normal PRs, so the tip of `vX-Y-test` is exactly what the beta should ship. -2. Run: +3. Run: ```shell script breeze release-management start-beta-process \ @@ -468,6 +507,9 @@ Before cutting an RC, we should look at the milestone and merge anything ready, The Release Candidate artifacts we vote upon should be the exact ones we vote against, without any modification other than renaming – i.e. the contents of the files must be the same between voted release candidate and final release. Because of this the version in the built artifacts that will become the official Apache releases must not include the rcN suffix. +**Before you build the artifacts**, run the upgrade check, as described in +[Before cutting a beta or an RC](#before-cutting-a-beta-or-an-rc-refresh-the-base-images-and-the-important-ci-versions). + > [!IMPORTANT] > When you start the release candidates of a major/minor release that went through beta releases > (`X.Y.0rc1`), `vX-Y-test` stops being fast-forwarded to `main` and changes reach it only by @@ -635,6 +677,11 @@ still works but is no longer recommended. - PR from the 'test' branch to the 'stable' branch + Before you open it, run `breeze workflow-run upgrade-check --branch vX-Y-test`, which pushes the + upgrade onto `vX-Y-test`, as described in + [Before cutting a beta or an RC](#before-cutting-a-beta-or-an-rc-refresh-the-base-images-and-the-important-ci-versions), + so the release is built on the latest base images. + Cherry-picked commits often include provider dependency bumps (changes to `>=` constraints on `apache-airflow-providers-*` packages in `pyproject.toml`). CI blocks such changes by default — only Release Managers should perform them. To allow the PR to pass, add the diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 2b69b30f11f..a0a2511193e 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -1159,6 +1159,42 @@ These are all available flags of ``workflow-run release-constraints`` command: :width: 100% :alt: Breeze workflow-run release-constraints +Refreshing the base images before a release +""""""""""""""""""""""""""""""""""""""""""" + +The images are built on hardened Python base images pinned to digests, and the legacy images on pinned +Python patch releases. Only the "Upgrade important CI environment" upgrade moves those pins, to images the +mirror already has, so the release manager runs it before every beta (for ``main``) and before every +test-to-stable PR (for the release branch), with one command. + +``breeze workflow-run upgrade-check`` mirrors the latest hardened base images first and waits for it, then +upgrades the branch - by default the branch breeze comes from. For ``main`` it runs the upgrade check +workflow, which opens an upgrade PR, and waits for it. For a ``vX-Y-test`` branch it runs +``breeze ci upgrade --push-to-target-branch`` in your checkout, which pushes the upgrade straight to the +branch, the head of the test-to-stable PR. That runs locally, as the release branch rules let only +committers push without CI having passed. It fails, rather than switching, resetting or cleaning anything, +when the checkout is not at the tip of the branch or has local changes. Both need ``gh`` credentials that +may dispatch ``apache/airflow`` workflows, and check that first. + +These are all available flags of ``workflow-run upgrade-check`` command: + +.. image:: ./images/output_workflow-run_upgrade-check.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_workflow-run_upgrade-check.svg + :width: 100% + :alt: Breeze workflow-run upgrade-check + +``breeze workflow-run mirror-base-images`` runs only the mirror on ``main`` and waits for it. The mirror is +shared by all branches, and ``main`` mirrors every Python version an active release branch still builds +on. The upgrade check workflow runs it in a job of its own, so the token that may dispatch workflows never +reaches the job running the upgraded tools. + +These are all available flags of ``workflow-run mirror-base-images`` command: + +.. image:: ./images/output_workflow-run_mirror-base-images.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_workflow-run_mirror-base-images.svg + :width: 100% + :alt: Breeze workflow-run mirror-base-images + Publishing the schema files to S3 """"""""""""""""""""""""""""""""" diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 4e14542c15d..657369de610 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -429,7 +429,7 @@ can be used for CI images: | Build argument | Default value | Description | |-----------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------| -| `BASE_IMAGE` | `ghcr.io/apache/airflow/base/python:-debian13-dev` | Mirror of the hardened base image Python comes from | +| `BASE_IMAGE` | `ghcr.io/apache/airflow/base/python:-debian13-dev@` | Mirror of the hardened base image Python comes from, pinned to a digest by `breeze ci upgrade` | | `PYTHON_MAJOR_MINOR_VERSION` | `3.11` | major/minor version of Python (should match base image) | | `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | | `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | diff --git a/dev/breeze/doc/images/output_ci_upgrade.svg b/dev/breeze/doc/images/output_ci_upgrade.svg index 6c88c7eea84..09b2d26b592 100644 --- a/dev/breeze/doc/images/output_ci_upgrade.svg +++ b/dev/breeze/doc/images/output_ci_upgrade.svg @@ -1,4 +1,4 @@ - +