From 6ca7a3be6b9c95551b53baa96f02e578243a5534 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Nov 2026 01:04:28 +0100 Subject: [PATCH] Build the PROD packages beside the CI image and constraints (#76261) The PROD image build first built the Airflow and provider wheels, but only after the CI image and generate-constraints had finished, although the wheels need neither. In a RunsOn pull request run that put them on the chain that ends the run: CI image to minute 9.1, constraints to 15.2, wheels 15.5 to 17.9, PROD image to 29.6, Kubernetes tests to 41.3. The CI workflows now build the wheels in their own job right after build-info and tell the PROD image build they are already uploaded; the other callers of prod-image-build.yml still build them as before. Generated-by: Claude Opus 5 --- .github/workflows/ci-amd.yml | 19 ++- .github/workflows/ci-arm.yml | 19 ++- .github/workflows/prod-image-build.yml | 96 +++---------- .github/workflows/prod-packages-build.yml | 129 ++++++++++++++++++ .../tests/ci/test_prek_cache_optimizations.py | 2 +- 5 files changed, 187 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/prod-packages-build.yml diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index e888fd41ef3..b42c001b78d 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -1130,9 +1130,23 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} + build-prod-packages: + name: Build PROD packages + needs: [build-info] + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + upload-package-artifact: "true" + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + build-prod-images: name: Build PROD images - needs: [build-info, build-ci-images, generate-constraints] + needs: [build-info, build-ci-images, build-prod-packages, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -1155,6 +1169,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + packages-prebuilt: "true" additional-prod-image-tests: name: "Additional PROD image tests" @@ -1614,6 +1629,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - ci-image-checks @@ -1682,6 +1698,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - check-timed-out-jobs diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index b7b16022938..4a2b31c4524 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -1006,9 +1006,23 @@ jobs: use-uv: ${{ needs.build-info.outputs.use-uv }} default-branch: ${{ needs.build-info.outputs.default-branch }} + build-prod-packages: + name: Build PROD packages + needs: [build-info] + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + upload-package-artifact: "true" + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + build-prod-images: name: Build PROD images - needs: [build-info, build-ci-images, generate-constraints] + needs: [build-info, build-ci-images, build-prod-packages, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -1031,6 +1045,7 @@ jobs: docker-cache: ${{ needs.build-info.outputs.docker-cache }} disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + packages-prebuilt: "true" additional-prod-image-tests: name: "Additional PROD image tests" @@ -1490,6 +1505,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - ci-image-checks @@ -1557,6 +1573,7 @@ jobs: - build-info - build-java-sdk-docs - build-prod-images + - build-prod-packages - build-ts-sdk-docs - check-run-needed - check-timed-out-jobs diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 5cac2e2817d..1df788fac5e 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -113,86 +113,28 @@ on: # yamllint disable-line rule:truthy description: "Whether this is a prod-image build (true/false)" required: true type: string + packages-prebuilt: + description: "Whether the caller already uploaded the prod-packages artifact (true/false)" + required: false + default: "false" + type: string permissions: contents: read jobs: build-prod-packages: name: "Build Airflow and provider distributions" - timeout-minutes: 10 - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} - if: inputs.prod-image-build == 'true' - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - steps: - - name: "Cleanup repo" - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - if: inputs.upload-package-artifact == 'true' - - name: "Checkout target branch" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: "Make /mnt writeable" - run: ./scripts/ci/make_mnt_writeable.sh - if: inputs.upload-package-artifact == 'true' - - name: "Move docker to /mnt" - run: ./scripts/ci/move_docker_to_mnt.sh - if: inputs.upload-package-artifact == 'true' - - name: "Cleanup dist and context file" - shell: bash - run: rm -fv ./dist/* ./docker-context-files/* - if: inputs.upload-package-artifact == 'true' - - name: "Install Breeze" - uses: ./.github/actions/breeze - if: inputs.upload-package-artifact == 'true' - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - platform: ${{ inputs.platform }} - save-cache: false - # The hatch_build.py files of the fab, edge3 and common.ai providers run these hooks. - hooks: compile-fab-assets compile-edge-assets compile-common-ai-provider-assets - if: inputs.upload-package-artifact == 'true' - - name: "Prepare providers packages - all providers built from sources" - shell: bash - run: > - breeze release-management prepare-provider-distributions - --distributions-list-file ./prod_image_installed_providers.txt - --distribution-format wheel --include-not-ready-providers --skip-tag-check - if: > - inputs.upload-package-artifact == 'true' && inputs.branch == 'main' - - name: "Prepare providers packages with only new versions of providers" - shell: bash - run: > - breeze release-management prepare-provider-distributions - --distributions-list-file ./prod_image_installed_providers.txt - --distribution-format wheel --include-not-ready-providers --github-repository apache/airflow - if: > - inputs.upload-package-artifact == 'true' && inputs.branch != 'main' - - name: "Prepare airflow package" - shell: bash - run: > - breeze release-management prepare-airflow-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Prepare task-sdk package" - shell: bash - run: > - breeze release-management prepare-task-sdk-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Prepare airflow-ctl package" - shell: bash - run: > - breeze release-management prepare-airflow-ctl-distributions --distribution-format wheel - if: inputs.upload-package-artifact == 'true' - - name: "Upload prepared packages as artifacts" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: prod-packages - path: ./dist - retention-days: 7 - if-no-files-found: error - if: inputs.upload-package-artifact == 'true' + # The CI workflows build the packages before calling this workflow; see prod-packages-build.yml. + if: inputs.packages-prebuilt != 'true' + uses: ./.github/workflows/prod-packages-build.yml + permissions: + contents: read + with: + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} + default-python-version: ${{ inputs.default-python-version }} + branch: ${{ inputs.branch }} + upload-package-artifact: ${{ inputs.upload-package-artifact }} + prod-image-build: ${{ inputs.prod-image-build }} build-prod-images: strategy: @@ -204,6 +146,10 @@ jobs: runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} needs: - build-prod-packages + # A skipped build-prod-packages means the caller built the packages already. + if: >- + !cancelled() && inputs.prod-image-build == 'true' && + (needs.build-prod-packages.result == 'success' || inputs.packages-prebuilt == 'true') env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" diff --git a/.github/workflows/prod-packages-build.yml b/.github/workflows/prod-packages-build.yml new file mode 100644 index 00000000000..c80bfd816db --- /dev/null +++ b/.github/workflows/prod-packages-build.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# Builds the Airflow, task-sdk, airflow-ctl and provider wheels the PROD image installs. It reads no +# constraints and no CI image, so the CI workflows start it right after build-info, beside the CI +# image build and generate-constraints, instead of as the first step of the PROD image build. +name: Build PROD packages +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string + default-python-version: + description: "Python version used to build the packages" + required: true + type: string + branch: + description: "Branch used to run the CI jobs in (main/v*_*_test)." + required: true + type: string + upload-package-artifact: + description: "Whether to build and upload the packages as the prod-packages artifact (true/false)" + required: true + type: string + prod-image-build: + description: "Whether this is a prod-image build (true/false)" + required: true + type: string +permissions: + contents: read +jobs: + build-prod-packages: + name: "Build Airflow and provider distributions" + timeout-minutes: 10 + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + if: inputs.prod-image-build == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + if: inputs.upload-package-artifact == 'true' + - name: "Checkout target branch" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: "Make /mnt writeable" + run: ./scripts/ci/make_mnt_writeable.sh + if: inputs.upload-package-artifact == 'true' + - name: "Move docker to /mnt" + run: ./scripts/ci/move_docker_to_mnt.sh + if: inputs.upload-package-artifact == 'true' + - name: "Cleanup dist and context file" + shell: bash + run: rm -fv ./dist/* ./docker-context-files/* + if: inputs.upload-package-artifact == 'true' + - name: "Install Breeze" + uses: ./.github/actions/breeze + if: inputs.upload-package-artifact == 'true' + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + platform: ${{ inputs.platform }} + save-cache: false + # The hatch_build.py files of the fab, edge3 and common.ai providers run these hooks. + hooks: compile-fab-assets compile-edge-assets compile-common-ai-provider-assets + if: inputs.upload-package-artifact == 'true' + - name: "Prepare providers packages - all providers built from sources" + shell: bash + run: > + breeze release-management prepare-provider-distributions + --distributions-list-file ./prod_image_installed_providers.txt + --distribution-format wheel --include-not-ready-providers --skip-tag-check + if: > + inputs.upload-package-artifact == 'true' && inputs.branch == 'main' + - name: "Prepare providers packages with only new versions of providers" + shell: bash + run: > + breeze release-management prepare-provider-distributions + --distributions-list-file ./prod_image_installed_providers.txt + --distribution-format wheel --include-not-ready-providers --github-repository apache/airflow + if: > + inputs.upload-package-artifact == 'true' && inputs.branch != 'main' + - name: "Prepare airflow package" + shell: bash + run: > + breeze release-management prepare-airflow-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Prepare task-sdk package" + shell: bash + run: > + breeze release-management prepare-task-sdk-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Prepare airflow-ctl package" + shell: bash + run: > + breeze release-management prepare-airflow-ctl-distributions --distribution-format wheel + if: inputs.upload-package-artifact == 'true' + - name: "Upload prepared packages as artifacts" + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: prod-packages + path: ./dist + retention-days: 7 + if-no-files-found: error + if: inputs.upload-package-artifact == 'true' diff --git a/scripts/tests/ci/test_prek_cache_optimizations.py b/scripts/tests/ci/test_prek_cache_optimizations.py index 8d8449153ce..6310637a751 100644 --- a/scripts/tests/ci/test_prek_cache_optimizations.py +++ b/scripts/tests/ci/test_prek_cache_optimizations.py @@ -270,7 +270,7 @@ def test_selected_hooks_match_the_hooks_each_job_runs(): "ci-arm.yml:migration-round-trip", "ci-arm.yml:mypy-providers", "generate-constraints.yml:generate-constraints-matrix", - "prod-image-build.yml:build-prod-packages", + "prod-packages-build.yml:build-prod-packages", "test-providers.yml:prepare-install-verify-provider-distributions", "test-providers.yml:providers-compatibility-tests-matrix", ]