nprasad2077/airflow
1
0 Fork 0
mirror of https://github.com/apache/airflow.git synced 2026-11-04 07:12:48 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
5d886dd7414db79d3388a58e1c1e072720ca6ec2
airflow/.github/workflows/ci-amd.yml
T

1885 lines
92 KiB
YAML
Raw Blame History

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
# AMD CI workflow. Triggered on every PR, on the canary cron, and on release branches by
# `release-branch-ci.yml`. Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the
# scheduled canary. The two workflows are physical copies of each other
# (GitHub Actions has no cross-file YAML include), differing only in their
# trigger surface, the hardcoded `platform` value in `build-info`'s outputs
# (and the `PLATFORM` selective checks pick the runner type for), and the
# architecture-specific print-platform job.
# Edit both files together when changing pipeline shape.
name: Tests (AMD)
on: # yamllint disable-line rule:truthy
schedule:
# AMD canary runs 2x/day (01:58, 13:58), interleaved with ARM's 2x/day (07:28, 19:28)
# in `ci-arm.yml` so a full-matrix canary still runs roughly every ~6h (alternating
# architecture) while halving the scheduled AMD compute. The `:58` vs ARM's `:28`
# offset keeps the two from competing for runners at exactly the same minute.
- cron: '58 1,13 * * *'
pull_request:
branches:
- main
- v[0-9]+-[0-9]+-test
- v[0-9]+-[0-9]+-stable
- providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+
- chart/v[0-9]+-[0-9]+x-test
- chart/v[0-9]+-[0-9]+x-stable
- airflow-ctl/v[0-9]+-[0-9]+-test
- airflow-ctl/v[0-9]+-[0-9]+-stable
types: [opened, reopened, synchronize, ready_for_review]
push:
# Every push triggers the workflow so that contributors get CI in their forks (AIP-120).
# In apache/airflow the `gate` job keeps the old behaviour: only pushes to the release-prep /
# providers branches in CANONICAL_PUSH_BRANCH_FILTERS (fork_ci_gate.py) run anything, and they
# run only the tests needed by the changes since the last successful run of the branch
# (selected as for a PR); other pushes produce a gate-only run. Full runs of these branches, on
# AMD and ARM, are dispatched by `release-branch-ci.yml`. `main` does not build on push; its
# full runs are the canary above.
branches:
- '**'
workflow_dispatch:
permissions:
# All other permissions are set to none by default
contents: read
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
JAVA_VERSION: '11'
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
VERBOSE: "true"
concurrency:
# Push runs have a group of their own per branch: a newer push cancels the older push run (the
# newer run then also tests the cancelled run's changes, see `check-run-needed`), and a push never
# cancels a full run of the branch.
group: >-
ci-amd-${{ github.event.pull_request.number || github.ref }}${{
github.event_name == 'push' && '-push' || '' }}
# A scheduled canary queues behind a run in progress instead of cancelling it: it may well be a
# manual run of the same commit, which the canary would otherwise cancel and then skip itself.
cancel-in-progress: ${{ github.event_name != 'schedule' }}
jobs:
check-run-needed:
name: "Check if the run is needed"
# A scheduled canary is not needed when the last completed full run of this workflow on the
# branch tested the same commit and succeeded: it would only repeat that run. A push run tests
# the changes since the last successful run of this workflow on the branch, found here.
# This job runs for every event, with no steps for the others: every other job depends on it
# through `build-info`, and GitHub skips the whole dependency chain below a skipped job unless
# each job in it has a status function such as `always()` in its `if:`.
runs-on: ["ubuntu-slim"]
timeout-minutes: 5
permissions:
actions: read
contents: read
outputs:
run-needed: ${{ steps.schedule.outputs.run-needed || 'true' }}
# Empty in forks, whose pushes selective checks compare against apache/airflow.
push-base-commit: ${{ steps.push.outputs.push-base-commit }}
steps:
- name: "Find the last successful run of the branch"
if: github.event_name == 'push' && github.repository == 'apache/airflow'
id: push
shell: bash
env:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
WORKFLOW_REF: ${{ github.workflow_ref }}
BRANCH: ${{ github.ref_name }}
run: |
set -euo pipefail
workflow_path="${WORKFLOW_REF%@*}"
# A successful push or full run tested everything up to its commit. Changes after it -- from
# pushes whose runs failed, or were cancelled by a newer push -- are all tested by this run.
# A successful run of this same commit is passed over, so that a re-run tests again.
if ! base="$(gh api "repos/${REPOSITORY}/actions/workflows/${workflow_path##*/}/runs" -X GET \
-f branch="${BRANCH}" -f status=success -F per_page=30 \
--jq "[.workflow_runs[] | select((.event == \"push\" or .event == \"workflow_dispatch\")
and .head_sha != \"${GITHUB_SHA}\")][0].head_sha // \"\"")"; then
# Fail open: a failed lookup must not skip the run.
echo "::warning::Could not find the last successful run of ${BRANCH}, all tests will run"
base=""
fi
# With no successful run to build on, selective checks run all tests.
echo "push-base-commit=${base}" >> "${GITHUB_OUTPUT}"
echo "Testing the changes since ${base:-the start: no successful run yet, so all tests run}." \
>> "${GITHUB_STEP_SUMMARY}"
- name: "Check the last full run of ${{ github.sha }}"
if: github.event_name == 'schedule'
id: schedule
shell: bash
env:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
WORKFLOW_REF: ${{ github.workflow_ref }}
BRANCH: ${{ github.ref_name }}
run: |
set -euo pipefail
# A fork that enables Actions for fork CI also gets this cron; its canary would only use
# the contributor's runners. `ci-amd.yml`'s gate skips it too (`fork_schedule`).
if [[ "${REPOSITORY}" != "apache/airflow" ]]; then
echo "run-needed=false" >> "${GITHUB_OUTPUT}"
echo "Scheduled canaries run only in apache/airflow." >> "${GITHUB_STEP_SUMMARY}"
exit 0
fi
workflow_path="${WORKFLOW_REF%@*}"
workflow="${workflow_path##*/}"
# Full runs are the scheduled canaries and manual `workflow_dispatch` runs; this run is
# still in progress, so `status=completed` leaves it out.
if ! last="$(gh api \
"repos/${REPOSITORY}/actions/workflows/${workflow}/runs?branch=${BRANCH}&status=completed" \
-F per_page=20 -X GET \
--jq '[.workflow_runs[] | select(.event == "schedule" or .event == "workflow_dispatch")][0]
| "\(.head_sha // "") \(.conclusion // "") \(.html_url // "")"')"; then
# Fail open: a canary that cannot check the last run runs, and reports to Slack.
echo "::warning::Could not find the last full run of ${BRANCH}, running the canary"
last=""
fi
read -r last_sha last_conclusion last_url <<< "${last}"
if [[ "${last_sha}" == "${GITHUB_SHA}" && "${last_conclusion}" == "success" ]]; then
echo "run-needed=false" >> "${GITHUB_OUTPUT}"
echo "No new commits since the last successful full run: ${last_url}" >> "${GITHUB_STEP_SUMMARY}"
else
echo "run-needed=true" >> "${GITHUB_OUTPUT}"
fi
gate:
# Fork CI gate (AIP-120): decides whether project CI runs for this event. Configuration and
# code come from the base branch so a pull request cannot change its own eligibility. See
# dev/breeze/doc/ci/08_fork_ci_for_external_contributors.md. A pull request whose CI runs
# in the contributor's fork and changes this job, how build-info depends on it, any job that
# runs when it skips build-info (such as ci-passed), or the name of any job is closed by the
# fork CI monitor ("Gate protection" in that document).
name: "Fork CI gate"
runs-on: ["ubuntu-slim"]
timeout-minutes: 5
permissions:
contents: read
# Reads the creation time of this run, which decides for a pilot's older pull request.
actions: read
outputs:
project-ci: ${{ steps.gate.outputs.project-ci }}
reason: ${{ steps.gate.outputs.reason }}
steps:
- name: "Checkout gate configuration from the base branch"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# A push in apache/airflow has no base branch. Its decision depends only on the pushed
# ref, so it takes the gate from the default branch: a pushed branch without the gate
# files would otherwise look like `base_without_gate` and run project CI on a push that
# never triggered it. A push in a fork keeps the pushed commit's gate, by SHA: the fork's
# default branch, or a later tip of the pushed branch picked up by a re-run, is the
# contributor's to change outside the commit whose result the reconciler trusts, while
# a gate change in the pushed commit shows in the pull request's files.
ref: >-
${{ github.base_ref ||
(github.event_name == 'push' && github.repository == 'apache/airflow' &&
github.event.repository.default_branch) ||
(github.event_name == 'push' && github.sha) ||
github.ref }}
fetch-depth: 1
persist-credentials: false
sparse-checkout: |
.github/fork-ci-gate.toml
dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py
scripts/ci/prek/check_no_conventional_commit_message.py
sparse-checkout-cone-mode: false
# Until the gate is on the default branch, a push in apache/airflow would find no gate there
# and run the full project CI as `base_without_gate`, for every branch pushed. Only
# committers can push to apache/airflow, so the pushed commit's own gate decides then.
- name: "Check whether the default branch has the gate"
id: default-branch-gate
if: github.event_name == 'push' && github.repository == 'apache/airflow'
env:
GATE_CONFIG: .github/fork-ci-gate.toml
GATE_SCRIPT: dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py
run: |
if [[ -e "${GATE_CONFIG}" || -e "${GATE_SCRIPT}" ]]; then
echo "present=true" >> "${GITHUB_OUTPUT}"
else
echo "present=false" >> "${GITHUB_OUTPUT}"
fi
- name: "Checkout gate configuration from the pushed commit"
if: steps.default-branch-gate.outputs.present == 'false'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
fetch-depth: 1
persist-credentials: false
sparse-checkout: |
.github/fork-ci-gate.toml
dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py
scripts/ci/prek/check_no_conventional_commit_message.py
sparse-checkout-cone-mode: false
- name: "Decide whether project CI runs"
id: gate
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GATE_CONFIG: .github/fork-ci-gate.toml
GATE_SCRIPT: dev/breeze/src/airflow_breeze/utils/fork_ci_gate.py
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
PR_AUTHOR_ASSOCIATION: ${{ github.event.pull_request.author_association }}
PR_CREATED_AT: ${{ github.event.pull_request.created_at }}
# A base branch with neither gate file predates the gate (the pull request adding it,
# or a release branch cut before it) and keeps running project CI. With only one of
# them present, the missing file fails the job instead of opening the gate.
run: |
if [[ ! -e "${GATE_CONFIG}" && ! -e "${GATE_SCRIPT}" ]]; then
echo "project-ci=true reason=base_without_gate"
echo "project-ci=true" >> "${GITHUB_OUTPUT}"
echo "reason=base_without_gate" >> "${GITHUB_OUTPUT}"
echo 'Project CI runs (fork CI gate reason: `base_without_gate`)' >> "${GITHUB_STEP_SUMMARY}"
exit 0
fi
python3 "${GATE_SCRIPT}"
# A gated pull request skips ci-image-checks.yml, which checks the title on project CI, and
# its fork run is a `push` event without a title.
- name: "Check PR title is not Conventional Commits"
if: github.event_name == 'pull_request' && steps.gate.outputs.project-ci == 'false'
env:
PR_TITLE: ${{ github.event.pull_request.title }}
run: |
printf '%s\n' "$PR_TITLE" \
| python3 scripts/ci/prek/check_no_conventional_commit_message.py --from-stdin
build-info:
name: "Build info"
needs: [gate, check-run-needed]
if: >-
needs.gate.outputs.project-ci == 'true' &&
needs.check-run-needed.outputs.run-needed == 'true'
# At build-info stage we do not yet have outputs so the runs-on is hard-coded. ubuntu-slim is enough:
# it only installs breeze and runs selective checks, which need neither docker nor freeing disk space.
runs-on: ["ubuntu-slim"]
outputs:
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
core-test-types-list-as-strings-in-json: >-
${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }}
excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }}
force-pip: ${{ steps.selective-checks.outputs.force-pip }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }}
helm-test-kubernetes-versions: ${{ steps.selective-checks.outputs.helm-test-kubernetes-versions }}
include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
individual-providers-test-types-list-as-strings-in-json: >-
${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }}
java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }}
kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
kubernetes-combos-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
kustomize-overlay-names: ${{ steps.selective-checks.outputs.kustomize-overlay-names }}
lang-sdk-conformance-sdks: ${{ steps.selective-checks.outputs.lang-sdk-conformance-sdks }}
run-kustomize-overlays-tests: ${{ steps.selective-checks.outputs.run-kustomize-overlays-tests }}
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
newsfragment-pr-number: ${{ steps.newsfragment-pr.outputs.pr-number }}
# platform is hardcoded per workflow file (this is the
# AMD copy). The matching ARM copy lives in ci-arm.yml.
platform: "linux/amd64"
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
# yamllint disable rule:line-length
providers-compatibility-tests-matrix: >
${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
providers-test-types-list-as-strings-in-json: >-
${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }}
providers-split-test-types-list-as-strings-in-json: >-
${{ steps.selective-checks.outputs.providers-split-test-types-list-as-strings-in-json }}
providers-xdist-test-types-list-as-strings-in-json: >-
${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
run-agent-framework-tests: ${{ steps.selective-checks.outputs.run-agent-framework-tests }}
run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }}
run-airflow-ctl-integration-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-integration-tests }}
run-generic-basic-tests: ${{ steps.selective-checks.outputs.run-generic-basic-tests }}
run-generic-prod-image-tests: ${{ steps.selective-checks.outputs.run-generic-prod-image-tests }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
run-api-codegen: ${{ steps.selective-checks.outputs.run-api-codegen }}
run-api-tests: ${{ steps.selective-checks.outputs.run-api-tests }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }}
run-java-sdk-tests: ${{ steps.selective-checks.outputs.run-java-sdk-tests }}
run-ts-sdk-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-tests }}
run-ts-sdk-docs: ${{ steps.selective-checks.outputs.run-ts-sdk-docs }}
run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }}
run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
run-remote-logging-opensearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }}
run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }}
run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }}
run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }}
run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }}
run-ts-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-e2e-tests }}
run-yaml-dag-e2e-tests: ${{ steps.selective-checks.outputs.run-yaml-dag-e2e-tests }}
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }}
run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }}
runner-type: ${{ steps.selective-checks.outputs.runner-type }}
in-memory-runner-type: ${{ steps.selective-checks.outputs.in-memory-runner-type }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
selected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.selected-providers-list-as-string }}
shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }}
skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }}
skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }}
source-head-repo: ${{ steps.source-run-info.outputs.head-repo }}
source-head-ref: ${{ steps.source-run-info.outputs.head-ref }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }}
testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }}
use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }}
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Fetch the last tested push commit"
# Selective checks of a push diff against this commit. A two-commit diff only needs the
# commit itself, not the history in between; if it cannot be fetched, all tests run.
if: needs.check-run-needed.outputs.push-base-commit != ''
shell: bash
env:
PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }}
run: |
git fetch --no-tags --depth=1 origin "${PUSH_BASE_COMMIT}" \
|| echo "::warning::Could not fetch ${PUSH_BASE_COMMIT}, all tests will run"
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
free-space: "false"
id: breeze
- name: "Save github context to file"
# The github context can be very large for PRs that touch many files (e.g. Dependabot
# bumps with dozens of packages). Passing it as an env var would exceed ARG_MAX and
# prevent subsequent steps from starting bash, so we write it to a file once here and
# let downstream breeze invocations read it via GITHUB_CONTEXT_INPUT.
#
# Zizmor flags this as template-injection because attacker-controllable fields in the
# github context (PR title, body, branch names) are pasted into a `run:` script. It is
# a false positive in this specific shape:
# * the substituted JSON sits inside a SINGLE-QUOTED heredoc (`<< '__GITHUB_CONTEXT_END__'`),
# which disables ALL bash expansion -- $(...), `...`, ${...}, and even `\` are literal,
# so no shell metacharacter in the JSON can be evaluated;
# * the only remaining escape is the delimiter `__GITHUB_CONTEXT_END__` appearing on a
# line by itself in the JSON. `toJson` escapes newlines inside string values as `\n`
# (so they are NOT real newlines in the script source), and the delimiter is unique
# enough that github will never produce it as a structural line of the JSON output;
# * `cat` writes the bytes verbatim to a file -- it is not eval'd.
# Whatever junk a malicious PR puts in title/body therefore lands inside a JSON file
# consumed by breeze, never executed as shell.
shell: bash
run: | # zizmor: ignore[template-injection]
cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__'
${{ toJson(github) }}
__GITHUB_CONTEXT_END__
- name: "Get information about the Workflow"
id: source-run-info
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
env:
SKIP_BREEZE_SELF_UPGRADE_CHECK: "true"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
# The newsfragment check of static checks fails a newsfragment of the pull request that is
# named after another number. A push in a fork (AIP-120) has no pull request in its event,
# so the open pull request of the pushed branch is looked up; without one, any number passes.
- name: "Find the pull request for the newsfragment check"
id: newsfragment-pr
env:
GH_TOKEN: ${{ github.token }}
EVENT_NAME: ${{ github.event_name }}
BASE_REF: ${{ github.base_ref }}
EVENT_PR_NUMBER: ${{ github.event.pull_request.number }}
REPOSITORY: ${{ github.repository }}
OWNER: ${{ github.repository_owner }}
REF_NAME: ${{ github.ref_name }}
run: |
number=""
# A pull request inside a fork has a number of the fork's, not of apache/airflow.
if [[ "${EVENT_NAME}" == "pull_request" && "${BASE_REF}" == "main" \
&& "${REPOSITORY}" == "apache/airflow" ]]; then
number="${EVENT_PR_NUMBER}"
elif [[ "${EVENT_NAME}" == "push" && "${REPOSITORY}" != "apache/airflow" ]]; then
number=$(gh api -X GET repos/apache/airflow/pulls -f head="${OWNER}:${REF_NAME}" \
-f state=open -f base=main --jq '.[0].number // empty' || true)
fi
if [[ "${number}" =~ ^[0-9]+$ ]]; then
echo "pr-number=${number}" >> "${GITHUB_OUTPUT}"
fi
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
# Repository variable: "true" moves every run, pushes included, back to public runners.
USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
# Repository variable: "false" keeps docker of the RunsOn jobs on disk instead of in RAM.
USE_IN_MEMORY_DOCKER: "${{ vars.USE_IN_MEMORY_DOCKER }}"
PLATFORM: "linux/amd64"
SELECTIVE_TESTS_ON_PUSH: "true"
PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }}
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: env
run: printenv
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
print-platform:
name: "Platform: AMD"
needs: [build-info]
runs-on: ["ubuntu-slim"]
steps:
- name: "Print architecture"
run: "echo '## Architecture: AMD' >> $GITHUB_STEP_SUMMARY"
basic-tests:
name: "Basic tests"
needs: [build-info, check-run-needed]
uses: ./.github/workflows/basic-tests.yml
with:
newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }}
runners: ${{ needs.build-info.outputs.runner-type }}
run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}}
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
canary-run: ${{needs.build-info.outputs.canary-run}}
run-breeze-integration-tests: ${{needs.build-info.outputs.run-breeze-integration-tests}}
run-scripts-tests: ${{needs.build-info.outputs.run-scripts-tests}}
run-generic-basic-tests: ${{needs.build-info.outputs.run-generic-basic-tests}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
use-uv: ${{needs.build-info.outputs.use-uv}}
platform: ${{ needs.build-info.outputs.platform }}
shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}}
push-base-commit: ${{ needs.check-run-needed.outputs.push-base-commit }}
build-ci-images:
name: Build CI images
needs: [build-info]
uses: ./.github/workflows/ci-image-build.yml
permissions:
contents: read
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
packages: write
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
push-image: "false"
upload-image-artifact: "true"
upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
if: needs.build-info.outputs.ci-image-build == 'true'
additional-ci-image-checks:
name: "Additional CI image checks"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/additional-ci-image-checks.yml
permissions:
contents: read
packages: write
id-token: write
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
branch: ${{ needs.build-info.outputs.default-branch }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
generate-constraints:
name: "Generate constraints"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/generate-constraints.yml
if: needs.build-info.outputs.ci-image-build == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
generate-pypi-constraints: "true"
# generate no providers constraints only in canary builds - they take quite some time to generate
# they are not needed for regular builds, they are only needed to update constraints in canaries
generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
secrets:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
ci-image-checks:
name: "CI image checks"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/ci-image-checks.yml
permissions:
id-token: write
contents: read
with:
newsfragment-pr-number: ${{ needs.build-info.outputs.newsfragment-pr-number }}
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }}
basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
docs-build: ${{ needs.build-info.outputs.docs-build }}
run-api-codegen: ${{ needs.build-info.outputs.run-api-codegen }}
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
source-head-repo: ${{ needs.build-info.outputs.source-head-repo }}
source-head-ref: ${{ needs.build-info.outputs.source-head-ref }}
secrets:
DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
mypy-providers:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
platform: ${{ needs.build-info.outputs.platform }}
python: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
make-mnt-writeable-and-cleanup: true
id: breeze
- name: "Install prek"
uses: ./.github/actions/install-prek
id: prek
with:
platform: ${{ needs.build-info.outputs.platform }}
save-cache: false
hooks: mypy-providers
- name: "MyPy checks for providers"
run: prek --color always --verbose --stage manual mypy-providers --all-files
env:
VERBOSE: "false"
COLUMNS: "202"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
INCLUDE_MYPY_VOLUME: "false"
migration-round-trip:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}"
uses: ./.github/actions/prepare_breeze_and_image
with:
platform: ${{ needs.build-info.outputs.platform }}
python: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
make-mnt-writeable-and-cleanup: true
id: breeze
- name: "Install prek"
uses: ./.github/actions/install-prek
id: prek
with:
platform: ${{ needs.build-info.outputs.platform }}
save-cache: false
hooks: migration-round-trip
- name: "Migration round-trip check"
run: prek --color always --verbose --stage manual migration-round-trip --all-files
env:
VERBOSE: "false"
COLUMNS: "202"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
providers:
name: "Provider distributions tests"
uses: ./.github/workflows/test-providers.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
if: >
needs.build-info.outputs.skip-providers-tests != 'true' &&
needs.build-info.outputs.latest-versions-only != 'true' &&
needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
# yamllint disable rule:line-length
providers-compatibility-tests-matrix: >
${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
providers-test-types-list-as-strings-in-json: >
${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
tests-helm:
name: "Helm tests"
uses: ./.github/workflows/helm-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
helm-test-kubernetes-versions: ${{ needs.build-info.outputs.helm-test-kubernetes-versions }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
if: >
needs.build-info.outputs.run-helm-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main' &&
needs.build-info.outputs.latest-versions-only != 'true'
tests-kustomize-overlays:
name: "Kustomize overlays smoke tests"
uses: ./.github/workflows/kustomize-overlays-tests.yml
needs: [build-info, build-prod-images]
permissions:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
overlays: ${{ needs.build-info.outputs.kustomize-overlay-names }}
# Gated by run-kustomize-overlays-tests (computed by SelectiveChecks
# from changes under chart/kustomize-overlays/ and the narrow set of
# runner files), NOT by run-helm-tests — a values.yaml tweak should
# not pull in a 30-40 min kind cluster spin-up for unrelated overlays.
# AMD-only via the platform guard below: the smoke test is
# CPU-architecture-neutral (kustomize render + apply + STATUS
# verify-block) and the overlays' third-party images (e.g.
# gcavalcante8808/krb5-server) are not published for arm64, so running it
# on the ARM lane would only add cost and arch-specific flakes for no
# extra signal. The job stays defined in both ci-arm.yml/ci-amd.yml (kept
# in sync) and is skipped on ARM through build-info.platform, mirroring
# the tests-unit-* jobs.
if: >
needs.build-info.outputs.run-kustomize-overlays-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main' &&
needs.build-info.outputs.latest-versions-only != 'true' &&
needs.build-info.outputs.kustomize-overlay-names != '[]' &&
needs.build-info.outputs.platform == 'linux/amd64'
tests-postgres-core:
name: "Postgres tests: core"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
test-group: "core"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.postgres-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-migration-tests: "true"
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-postgres-providers:
name: "Postgres tests: providers"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
test-scope: "DB"
test-group: "providers"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.postgres-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-migration-tests: "true"
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-mysql-core:
name: "MySQL tests: core"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
test-group: "core"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.mysql-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
job-timeout-minutes: 80
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64'
tests-mysql-providers:
name: "MySQL tests: providers"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
test-scope: "DB"
test-group: "providers"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.mysql-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64'
tests-sqlite-core:
name: "Sqlite tests: core"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
test-scope: "DB"
test-group: "core"
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for sqlite
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-sqlite-providers:
name: "Sqlite tests: providers"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
test-name-separator: ""
test-scope: "DB"
test-group: "providers"
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for sqlite
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
run-migration-tests: "true"
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-non-db-core:
name: "Non-DB tests: core"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
test-scope: "Non-DB"
test-group: "core"
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for non-db
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-non-db-providers:
name: "Non-DB tests: providers"
uses: ./.github/workflows/run-unit-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
test-name-separator: ""
test-scope: "Non-DB"
test-group: "providers"
python-versions: ${{ needs.build-info.outputs.python-versions }}
# No versions for non-db
backend-versions: "['']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.providers-xdist-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-special:
name: "Special tests"
uses: ./.github/workflows/special-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
if: >
needs.build-info.outputs.run-unit-tests == 'true' &&
(needs.build-info.outputs.canary-run == 'true' ||
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' ||
needs.build-info.outputs.full-tests-needed == 'true')
with:
default-branch: ${{ needs.build-info.outputs.default-branch }}
runners: ${{ needs.build-info.outputs.runner-type }}
in-memory-runners: ${{ needs.build-info.outputs.in-memory-runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
core-test-types-list-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
providers-test-types-list-as-strings-in-json: >
${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }}
providers-split-test-types-list-as-strings-in-json: >
${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
tests-integration-system:
name: Integration and System Tests
needs: [build-info, build-ci-images]
uses: ./.github/workflows/integration-system-tests.yml
permissions:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }}
testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }}
run-system-tests: ${{ needs.build-info.outputs.run-system-tests }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
if: needs.build-info.outputs.run-unit-tests == 'true'
tests-with-lowest-direct-resolution-core:
name: "Low dep tests:core"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/run-unit-tests.yml
permissions:
contents: read
packages: read
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
test-scope: "All"
test-group: "core"
backend: "sqlite"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']"
excluded-providers-as-string: ""
excludes: "[]"
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
monitor-delay-time-in-seconds: 120
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
tests-with-lowest-direct-resolution-providers:
name: "Low dep tests: providers"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/run-unit-tests.yml
permissions:
contents: read
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
test-scope: "All"
test-group: "providers"
backend: "sqlite"
python-versions: ${{ needs.build-info.outputs.python-versions }}
backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']"
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
excludes: "[]"
test-types-as-strings-in-json: >
${{ needs.build-info.outputs.individual-providers-test-types-list-as-strings-in-json }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
run-coverage: ${{ needs.build-info.outputs.run-coverage }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
monitor-delay-time-in-seconds: 120
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
build-prod-images:
name: Build PROD images
needs: [build-info, build-ci-images, generate-constraints]
uses: ./.github/workflows/prod-image-build.yml
permissions:
contents: read
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
packages: write
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
build-type: "Regular"
push-image: "false"
upload-image-artifact: "true"
upload-package-artifact: "true"
python-versions: ${{ needs.build-info.outputs.python-versions }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
branch: ${{ needs.build-info.outputs.default-branch }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
prod-image-build: ${{ needs.build-info.outputs.prod-image-build }}
additional-prod-image-tests:
name: "Additional PROD image tests"
needs: [build-info, build-prod-images, generate-constraints]
uses: ./.github/workflows/additional-prod-image-tests.yml
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
default-branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
run-task-sdk-integration-tests: ${{ needs.build-info.outputs.run-task-sdk-integration-tests }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
run-remote-logging-elasticsearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }}
run-remote-logging-opensearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }}
run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }}
run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }}
# yamllint disable rule:line-length
providers-compatibility-tests-matrix: >
${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
run-ts-sdk-e2e-tests: ${{ needs.build-info.outputs.run-ts-sdk-e2e-tests }}
run-yaml-dag-e2e-tests: ${{ needs.build-info.outputs.run-yaml-dag-e2e-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}
run-generic-prod-image-tests: ${{ needs.build-info.outputs.run-generic-prod-image-tests }}
if: needs.build-info.outputs.prod-image-build == 'true'
tests-kubernetes:
name: "Kubernetes tests"
uses: ./.github/workflows/k8s-tests.yml
needs: [build-info, build-prod-images]
permissions:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }}
java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
lang-sdk-kubernetes-combo: >-
${{ needs.build-info.outputs.default-python-version }}-${{ needs.build-info.outputs.default-kubernetes-version }}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.run-helm-tests == 'true')
tests-task-sdk:
name: "Task SDK tests"
uses: ./.github/workflows/airflow-distributions-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
distribution-name: "task-sdk"
distribution-cmd-format: "prepare-task-sdk-distributions"
test-type: "task-sdk-tests"
use-local-venv: 'false'
test-timeout: 20
if: needs.build-info.outputs.run-task-sdk-tests == 'true'
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
packages: read
if: needs.build-info.outputs.run-go-sdk-tests == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
VERBOSE: "true"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
# keep this in sync with go.mod in go-sdk/
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
# Pinned to the patch: actions/go-versions trails Go security releases by weeks, and
# setup-go resolves from that manifest whenever it can match, so a bare 1.25 lands on a
# toolchain whose standard library still carries advisories the scan below fails on.
go-version: '1.25.13'
cache-dependency-path: go-sdk/go.sum
# keep this in sync with go.mod in go-sdk/
- name: Setup Gotestsum
shell: bash
run: |
go install gotest.tools/gotestsum@c4a0df2e75a225d979a444342dd3db752b53619f # v1.13.0
gotestsum --version
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: Run Go tests
working-directory: ./go-sdk
run: gotestsum --format github-actions ./...
# Without -race, the Go SDK tests for concurrent use can still pass when a lock is missing.
# The plain run above has to stay because the race build leaves out files with `//go:build !race`.
- name: Run Go tests with the race detector
working-directory: ./go-sdk
run: gotestsum --format github-actions -- -race ./...
# Nothing else builds for a 32-bit `int` (386, arm, mipsle), where 2^32-1 constants overflow
# and int64 values wrap. Vet type-checks on any host; the tests need one that runs 386.
- name: Vet Go SDK for a 32-bit target
working-directory: ./go-sdk
env:
GOARCH: "386"
run: go vet ./...
- name: Run Go tests on a 32-bit target
if: runner.arch == 'X64'
working-directory: ./go-sdk
env:
GOARCH: "386"
run: gotestsum --format github-actions ./...
# Go modules have no central registry that pre-scans releases, so we scan our own
# dependency graph against the Go vulnerability database. govulncheck is reachability
# based, so it only fails on advisories that actually affect code we call. The version
# tag is immutable (Go modules are content-addressed via the checksum database).
- name: Scan Go dependencies for known vulnerabilities
working-directory: ./go-sdk
run: |
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
govulncheck ./...
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }}
timeout-minutes: 15
permissions:
contents: read
if: needs.build-info.outputs.run-ts-sdk-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
package_json_file: ts-sdk/package.json
run_install: false
# keep this in sync with the Node version in ts-sdk-release.yml
- name: Setup node
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24.19.0"
cache: 'pnpm'
cache-dependency-path: ts-sdk/pnpm-lock.yaml
- name: Install dependencies
working-directory: ./ts-sdk
run: pnpm install --frozen-lockfile
- name: Run TypeScript SDK tests
working-directory: ./ts-sdk
run: pnpm test
env:
FORCE_COLOR: 2
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }}
timeout-minutes: 15
permissions:
contents: read
packages: read
if: needs.build-info.outputs.run-java-sdk-tests == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
VERBOSE: "true"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup Java
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
with:
distribution: 'temurin'
java-version: ${{ env.JAVA_VERSION }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: Run Java SDK tests
working-directory: ./java-sdk
run: ./gradlew test
tests-lang-sdk-conformance:
name: "Lang SDK serialization conformance (${{ matrix.sdk }})"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }}
timeout-minutes: 20
permissions:
contents: read
strategy:
fail-fast: false
matrix:
sdk: ${{ fromJSON(needs.build-info.outputs.lang-sdk-conformance-sdks) }}
if: needs.build-info.outputs.lang-sdk-conformance-sdks != '[]'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
# Serializes the same Dags with Airflow, from airflow-core's sources, to compare them.
- name: "Install uv"
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: "${{ needs.build-info.outputs.default-python-version }}"
- name: Setup Go
if: matrix.sdk == 'go'
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go-sdk/go.mod
cache-dependency-path: go-sdk/go.sum
- name: Setup Java
if: matrix.sdk == 'java'
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
with:
distribution: 'temurin'
java-version: ${{ env.JAVA_VERSION }}
- name: Setup pnpm
if: matrix.sdk == 'ts'
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
package_json_file: ts-sdk/package.json
run_install: false
- name: Setup node
if: matrix.sdk == 'ts'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: "24.19.0"
cache: 'pnpm'
cache-dependency-path: ts-sdk/pnpm-lock.yaml
- name: "Check the ${{ matrix.sdk }} SDK serializes Dags the way Airflow does"
env:
SDK: ${{ matrix.sdk }}
run: python3 "scripts/ci/lang_sdk_serialization/check_${SDK}_sdk.py"
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }}
timeout-minutes: 30
permissions:
contents: read
packages: read
if: needs.build-info.outputs.run-java-sdk-tests == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
VERBOSE: "true"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Build Java SDK Javadoc"
run: breeze build-docs --sdk-docs-only --sdk=java
- name: "Upload Javadoc artifact"
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: java-sdk-docs
path: generated/_build/docs/java-sdk/
retention-days: 7
if-no-files-found: error
overwrite: 'true'
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }}
timeout-minutes: 30
permissions:
contents: read
packages: read
if: needs.build-info.outputs.run-ts-sdk-docs == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USERNAME: ${{ github.actor }}
VERBOSE: "true"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Build TypeScript SDK API reference"
run: breeze build-docs --sdk-docs-only --sdk=typescript
- name: "Upload TypeScript SDK docs artifact"
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: ts-sdk-docs
path: generated/_build/docs/ts-sdk/
retention-days: 7
if-no-files-found: error
overwrite: 'true'
tests-airflow-ctl:
name: "Airflow CTL tests"
uses: ./.github/workflows/airflow-distributions-tests.yml
needs: [build-info]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '4+8', '16+32') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
distribution-name: "airflow-ctl"
distribution-cmd-format: "prepare-airflow-ctl-distributions"
test-type: "airflow-ctl-tests"
use-local-venv: 'true'
test-timeout: 20
if: needs.build-info.outputs.run-airflow-ctl-tests == 'true'
tests-agent-frameworks:
name: "Agent framework tests"
uses: ./.github/workflows/agent-framework-tests.yml
needs: [build-info, build-ci-images]
permissions:
contents: read
packages: read
with:
runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
# Not a dependency of finalize-tests: on a canary run the frameworks' own pins win, and a
# breaking framework release should not stop finalize-tests from pushing the image cache.
if: needs.build-info.outputs.run-agent-framework-tests == 'true'
finalize-tests:
name: Finalize tests
permissions:
contents: write
packages: write
# This will fire when all the jobs from "needs" are either successful or skipped
# build-info is skipped by a no-op canary and when the fork CI gate leaves the pull request to
# fork CI (AIP-120); every other need is then skipped too, which always() alone would treat as
# success.
if: always() && !failure() && !cancelled() && needs.build-info.result == 'success'
needs:
- additional-ci-image-checks
- additional-prod-image-tests
- basic-tests
- build-info
- build-prod-images
- ci-image-checks
- generate-constraints
- migration-round-trip
- mypy-providers
- providers
- tests-helm
- tests-integration-system
- tests-kubernetes
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
- tests-non-db-providers
- tests-postgres-core
- tests-postgres-providers
- tests-sqlite-core
- tests-sqlite-providers
- tests-task-sdk
- tests-airflow-ctl
- tests-go-sdk
- tests-java-sdk
- tests-ts-sdk
- tests-lang-sdk-conformance
- build-java-sdk-docs
- build-ts-sdk-docs
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
uses: ./.github/workflows/finalize-tests.yml
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
check-timed-out-jobs:
# A job-level `if` is the only place `cancelled()` tells a cancelled run from a job cancelled by
# its timeout: the steps of `ci-passed` start after the cancellation. Its `needs` must match
# those of `ci-passed`, so a timeout in any job fails the gate.
name: "Check for timed-out CI jobs"
if: always() && !cancelled() && contains(needs.*.result, 'cancelled')
needs:
- additional-ci-image-checks
- additional-prod-image-tests
- basic-tests
- build-ci-images
- build-info
- build-java-sdk-docs
- build-prod-images
- build-ts-sdk-docs
- check-run-needed
- ci-image-checks
- finalize-tests
- gate
- generate-constraints
- migration-round-trip
- mypy-providers
- print-platform
- providers
- tests-agent-frameworks
- tests-airflow-ctl
- tests-go-sdk
- tests-helm
- tests-integration-system
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
- tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
- tests-non-db-providers
- tests-postgres-core
- tests-postgres-providers
- tests-special
- tests-sqlite-core
- tests-sqlite-providers
- tests-task-sdk
- tests-ts-sdk
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
runs-on: ["ubuntu-slim"]
steps:
- name: "Fail because a CI job was cancelled without the run being cancelled"
run: |
echo "A CI job was cancelled, likely by its timeout, while the run was not cancelled."
exit 1
ci-passed:
# A skipped build-info means project CI did not run at all: the fork CI gate (AIP-120) left the
# pull request to CI in the contributor's fork, and every job here is skipped. Reporting
# "CI passed" then would let it merge on no CI, so the job reports under another name and
# the fork CI reconciler posts "CI passed" from the fork run instead.
# A run with cancelled jobs and no failed one -- usually a duplicate run of the same commit that
# concurrency cancelled, such as the two `synchronize` events GitHub sends when a stacked pull
# request's base and head are force-pushed together -- reports under another name too, so it
# never puts a failed "CI passed" next to the passing one of its twin. "CI passed" is then
# missing for that run, which keeps the merge blocked until another run of the commit reports it.
# A job cancelled by its timeout, while the run is not, makes `check-timed-out-jobs` fail, and
# so this job fails.
name: >-
${{ (!contains(needs.*.result, 'failure') && contains(needs.*.result, 'cancelled') &&
'CI passed (run cancelled)') ||
(needs.build-info.result == 'skipped' && 'CI passed (project CI not run)') || 'CI passed' }}
# The one check main's ruleset requires, so auto-merge waits for CI to finish. GitHub treats a
# skipped required check as passed, so this job always runs and fails itself when any job it
# needs failed or timed out. Jobs skipped by selective checks count as passed.
if: always()
needs:
- additional-ci-image-checks
- additional-prod-image-tests
- basic-tests
- build-ci-images
- build-info
- build-java-sdk-docs
- build-prod-images
- build-ts-sdk-docs
- check-run-needed
- check-timed-out-jobs
- ci-image-checks
- finalize-tests
- gate
- generate-constraints
- migration-round-trip
- mypy-providers
- print-platform
- providers
- tests-agent-frameworks
- tests-airflow-ctl
- tests-go-sdk
- tests-helm
- tests-integration-system
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
- tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
- tests-non-db-providers
- tests-postgres-core
- tests-postgres-providers
- tests-special
- tests-sqlite-core
- tests-sqlite-providers
- tests-task-sdk
- tests-ts-sdk
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
runs-on: ["ubuntu-slim"]
steps:
- name: "Fail if any CI job failed"
if: contains(needs.*.result, 'failure')
run: |
echo "At least one CI job failed."
exit 1
- name: "Report that CI jobs were cancelled"
if: contains(needs.*.result, 'cancelled')
run: |
echo "At least one CI job was cancelled, so this run does not report CI passed."
echo 'CI jobs were cancelled: this run does not report `CI passed`.' >> "${GITHUB_STEP_SUMMARY}"
- name: "All CI jobs passed or were skipped"
if: "!contains(needs.*.result, 'cancelled')"
run: echo "All CI jobs passed or were skipped."
notify-slack:
name: "Notify Slack"
needs:
- build-info
- check-run-needed
# Waits for the gate job, which also fails when a CI job timed out.
- ci-passed
- finalize-tests
- tests-agent-frameworks
# A no-op canary (`run-needed` is 'false') does not post; a failed check does. A failing gate
# skips build-info and everything after it, and must still be reported. A fork's scheduled run
# is the intended skip, and forks have no Slack to report to.
if: >-
always() &&
!cancelled() &&
github.repository == 'apache/airflow' &&
github.event_name == 'schedule' &&
needs.check-run-needed.outputs.run-needed != 'false'
runs-on: ["ubuntu-slim"]
permissions:
# The run's jobs, the previous notification state, and the annotations RunsOn puts on jobs
# stopped by an EC2 Spot interruption.
actions: read
checks: read
contents: read
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# Skips re-runs started by maintainers, and failures that are only EC2 Spot interruptions
# RunsOn will re-run; those leave the notification state alone, and the re-run reports.
- name: "Get failing jobs"
id: get-failures
shell: bash
run: python3 scripts/ci/spot_interruptions.py slack
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
- name: "Determine notification action"
id: notification
if: steps.get-failures.outputs.skip != 'true'
shell: bash
run: |
command -v gh >/dev/null 2>&1 || { echo "::error::gh CLI is not installed"; exit 1; }
gh auth status || gh auth login --with-token <<< "${GITHUB_TOKEN}"
python3 scripts/ci/slack_notification_state.py
env:
ARTIFACT_NAME: "slack-state-tests-${{ github.ref_name }}-amd"
CURRENT_FAILURES: "${{ steps.get-failures.outputs.failed-jobs }}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Upload notification state"
if: steps.get-failures.outputs.skip != 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: "slack-state-tests-${{ github.ref_name }}-amd"
path: ./slack-state/
retention-days: 7
overwrite: true
- name: "Notify Slack (new/changed failures)"
if: steps.notification.outputs.action == 'notify_new'
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
method: chat.postMessage
token: ${{ env.SLACK_BOT_TOKEN }}
# yamllint disable rule:line-length
payload: |
channel: "internal-airflow-ci-cd"
text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs-text }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
blocks:
- type: "section"
text:
type: "mrkdwn"
text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs-text }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
# yamllint enable rule:line-length
- name: "Notify Slack (still not fixed)"
if: steps.notification.outputs.action == 'notify_reminder'
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
method: chat.postMessage
token: ${{ env.SLACK_BOT_TOKEN }}
# yamllint disable rule:line-length
payload: |
channel: "internal-airflow-ci-cd"
text: "🚨🔁 Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs-text }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
blocks:
- type: "section"
text:
type: "mrkdwn"
text: "🚨🔁 Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs-text }}\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
# yamllint enable rule:line-length
- name: "Notify Slack (all tests passing)"
if: steps.notification.outputs.action == 'notify_recovery'
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
method: chat.postMessage
token: ${{ env.SLACK_BOT_TOKEN }}
# yamllint disable rule:line-length
payload: |
channel: "internal-airflow-ci-cd"
text: "✅ All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the run log>"
blocks:
- type: "section"
text:
type: "mrkdwn"
text: "✅ All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the run log>"
# yamllint enable rule:line-length
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
needs:
- build-info
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
- tests-non-db-providers
- tests-postgres-core
- tests-postgres-providers
- tests-sqlite-core
- tests-sqlite-providers
- tests-task-sdk
- tests-airflow-ctl
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
shell: bash
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
- name: "Download all test warning artifacts from the current build"
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
path: ./artifacts
pattern: test-warnings-*
- name: "Setup python"
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "${{ inputs.default-python-version }}"
- name: "Summarize all warnings"
run: |
./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \
--pattern "**/warnings-*.txt" \
--output ./files
- name: "Upload artifact for summarized warnings"
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: "test-summarized-warnings"
path: ./files/warn-summary-*.txt
retention-days: 7
if-no-files-found: ignore
overwrite: true
Reference in new issue View Git Blame Copy Permalink