nprasad2077/airflow
1
0 Fork 0
mirror of https://github.com/apache/airflow.git synced 2026-11-04 07:12:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run maintainer and canary CI on self-hosted RunsOn runners (#74889)

Browse Source 
This PR implements AIP-118 for integrating self-hosted runners in Apache Airflow's CI, utilizing the project's AWS instance and runs-on.com's runners.
This commit is contained in:
Shahar Epstein authored and GitHub committed 2026-10-25 10:48:19 +02:00
1 parent 054d093651
commit b5bc253fc2
33 files changed
+1110 -150
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/actions/breeze/action.yml
+2
View File
@@ -29,6 +29,8 @@ outputs:
runs:
using: "composite"
steps:
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Setup python"
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
.github/actions/prepare_breeze_and_image/action.yml
+3 -2
View File
@@ -82,12 +82,13 @@ runs:
- name: >
Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}
${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }}
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0
uses: ./.github/actions/stash-restore
with:
key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\
${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}"
path: "/mnt/"
# Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
# stashes as S3 digests only.
only-current-branch: 'true'
fail-on-download: 'true'
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
.github/actions/prepare_single_ci_image/action.yml
+3 -2
View File
@@ -36,11 +36,12 @@ runs:
using: "composite"
steps:
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0
uses: ./.github/actions/stash-restore
with:
key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}
path: "/mnt/"
# Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
# stashes as S3 digests only.
only-current-branch: 'true'
fail-on-download: 'true'
if: contains(inputs.python-versions-list-as-string, inputs.python)
.github/actions/runs-on-cost-summary/action.yml
+30
View File
@@ -0,0 +1,30 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job
# that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze.
name: 'RunsOn cost summary'
description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.'
runs:
using: 'composite'
steps:
- name: "RunsOn cost summary"
if: runner.environment == 'self-hosted'
uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
with:
show_costs: summary
.github/actions/stash-restore/action.yml
+137
View File
@@ -0,0 +1,137 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On RunsOn (the only
# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's in-region S3 cache
# bucket instead of leaving AWS as GitHub artifacts.
#
# Every RunsOn job can write anywhere in that bucket, so the bucket is not trusted: objects are
# named by their sha256, and the digest itself travels as a small ASF stash artifact, which
# GitHub scopes to the run / branch exactly like a regular stash. A blob that does not match the
# digest the ASF restore found for this run, branch and repository is never extracted. The bucket's
# lifecycle rule (S3CacheExpirationInDays) must outlive the longest retention-days of a digest
# (6 days today), or a digest hit finds no blob.
name: 'Restore stash'
description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn runners.'
inputs:
key:
description: 'Stash key.'
required: true
path:
description: >
Directory to restore into (GitHub-hosted only; the S3 tarball restores entries to the
paths they were saved from, and the restore then fails if any of them is outside this directory).
required: false
default: ${{ github.workspace }}
only-current-branch:
description: 'Do not fall back to the base branch stash.'
required: false
default: 'false'
fail-on-download:
description: 'Fail when a stash was found but could not be restored.'
required: false
default: 'false'
outputs:
stash-hit:
description: "'true' if a stash was restored, 'false' otherwise."
value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit }}
runs:
using: 'composite'
steps:
- name: "Restore stash from GitHub artifacts"
id: artifact
if: runner.environment != 'self-hosted'
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
with:
key: ${{ inputs.key }}
path: ${{ inputs.path }}
only-current-branch: ${{ inputs.only-current-branch }}
fail-on-download: ${{ inputs.fail-on-download }}
- name: "Restore stash digest from GitHub artifacts"
id: digest
if: runner.environment == 'self-hosted'
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
with:
key: ${{ inputs.key }}-s3
path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}
only-current-branch: ${{ inputs.only-current-branch }}
fail-on-download: ${{ inputs.fail-on-download }}
- name: "Restore stash from S3"
id: s3
if: runner.environment == 'self-hosted'
shell: bash
env:
STASH_KEY: ${{ inputs.key }}
STASH_PATH: ${{ inputs.path }}
DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }}
DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}/sha256
FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }}
run: |
set -uo pipefail
payload="${RUNNER_TEMP}/stash-restore-$$.tar"
entries="${RUNNER_TEMP}/stash-restore-$$.entries"
miss() {
rm -f "${payload}" "${entries}"
echo "stash-hit=false" >> "${GITHUB_OUTPUT}"
if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true" ]]; then
echo "::error::$1"
exit 1
fi
echo "$1"
exit 0
}
if [[ "${DIGEST_HIT}" != "true" ]]; then
miss "No stash '${STASH_KEY}'"
fi
digest="$(cat "${DIGEST_FILE}")"
if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
miss "Malformed digest for stash '${STASH_KEY}'"
fi
bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"
if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then
miss "Could not download ${object} for stash '${STASH_KEY}'"
fi
if ! echo "${digest} ${payload}" | sha256sum --check --status; then
miss "${object} does not match the digest recorded for stash '${STASH_KEY}'"
fi
# GNU tar detects the compression stash-save chose.
if ! tar -xPvf "${payload}" > "${entries}"; then
miss "Could not extract ${object} for stash '${STASH_KEY}'"
fi
rm -f "${payload}"
# The S3 tarball ignores `path`: fail loudly if a caller saves and restores different
# directories, which would work on GitHub-hosted runners only.
restore_dir="$(realpath -ms "${STASH_PATH}")"
restore_dir="${restore_dir%/}"
outside=()
while IFS= read -r entry; do
[[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}"
[[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}")
done < "${entries}"
rm -f "${entries}"
if [[ ${#outside[@]} -gt 0 ]]; then
printf '%s\n' "${outside[@]}"
echo "::error::Stash '${STASH_KEY}' restored the entries above outside of ${restore_dir}/"
exit 1
fi
echo "stash-hit=true" >> "${GITHUB_OUTPUT}"
echo "Restored stash '${STASH_KEY}' from ${object}"
.github/actions/stash-save/action.yml
+129
View File
@@ -0,0 +1,129 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn the tarball goes
# to the stack's S3 cache bucket under its sha256 and only the digest is saved as an ASF stash
# artifact. See stash-restore for why.
name: 'Save stash'
description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.'
inputs:
key:
description: 'Stash key.'
required: true
path:
description: 'File, directory or glob to stash.'
required: true
if-no-files-found:
description: "'error' fails when path matches nothing, 'warn' or 'ignore' do not."
required: false
default: 'warn'
retention-days:
description: >
Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's
S3CacheExpirationInDays lifecycle rule expires the tarball itself.
required: false
default: '5'
compression-level:
description: "Zlib compression level (0-9). On RunsOn, '0' stores the tarball uncompressed."
required: false
default: '6'
runs:
using: 'composite'
steps:
- name: "Save stash to GitHub artifacts"
if: runner.environment != 'self-hosted'
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
with:
key: ${{ inputs.key }}
path: ${{ inputs.path }}
if-no-files-found: ${{ inputs.if-no-files-found }}
retention-days: ${{ inputs.retention-days }}
compression-level: ${{ inputs.compression-level }}
- name: "Save stash to S3"
id: s3
if: runner.environment == 'self-hosted'
shell: bash
env:
STASH_KEY: ${{ inputs.key }}
STASH_PATH: ${{ inputs.path }}
DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }}
COMPRESSION_LEVEL: ${{ inputs.compression-level }}
run: |
set -euo pipefail
bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
shopt -s nullglob
# shellcheck disable=SC2206
candidates=( ${STASH_PATH} )
existing=()
for p in "${candidates[@]}"; do
if [[ -e "${p}" ]]; then
existing+=("${p}")
fi
done
if [[ ${#existing[@]} -eq 0 ]]; then
if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then
echo "::error::No files found for path '${STASH_PATH}'"
exit 1
fi
if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then
echo "::warning::No files found for path '${STASH_PATH}', skipping save"
fi
exit 0
fi
# The tarball is streamed to S3 while being hashed: a multi-GB image never touches the
# runner's disk. Its key is only known at the end, so it lands under a temporary key and
# is copied server-side to its digest key.
prefix="s3://${bucket}/cache/airflow-stash"
upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar"
digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo"
digest_file="${RUNNER_TEMP}/stash-save-$$.sha256"
trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT
mkfifo "${digest_fifo}"
sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" &
digest_pid=$!
compression=()
if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then
compression=(-z)
fi
tar -cP "${compression[@]}" -f - "${existing[@]}" \
| tee "${digest_fifo}" \
| aws s3 cp --only-show-errors - "${upload}"
wait "${digest_pid}"
digest="$(cat "${digest_file}")"
if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
echo "::error::Could not hash the stash '${STASH_KEY}'"
exit 1
fi
object="${prefix}/sha256/${digest}.tar"
aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}"
# Leaving the upload behind only costs storage until the bucket's lifecycle rule expires it.
aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not remove ${upload}"
mkdir -p "${DIGEST_DIR}"
echo "${digest}" > "${DIGEST_DIR}/sha256"
echo "saved=true" >> "${GITHUB_OUTPUT}"
echo "Saved ${existing[*]} to ${object}"
- name: "Save stash digest to GitHub artifacts"
if: runner.environment == 'self-hosted' && steps.s3.outputs.saved == 'true'
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
with:
key: ${{ inputs.key }}-s3
path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
if-no-files-found: error
retention-days: ${{ inputs.retention-days }}
.github/workflows/additional-ci-image-checks.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
# layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners.
timeout-minutes: 35
name: Check that image builds quickly
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
.github/workflows/additional-prod-image-tests.yml
+15 -15
View File
@@ -152,7 +152,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of PROD image building"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -189,7 +189,7 @@ jobs:
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker Compose quick start with PROD image verifying"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -220,7 +220,7 @@ jobs:
task-sdk-integration-tests:
timeout-minutes: 60
name: "Task SDK integration tests with PROD image"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -254,7 +254,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Regular e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -264,7 +264,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Remote logging e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -276,7 +276,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Elasticsearch remote logging e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -288,7 +288,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenSearch remote logging e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -300,7 +300,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "XCom object storage backend e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -311,7 +311,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Event driven e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -323,7 +323,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Java SDK e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -335,7 +335,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "Go SDK e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -347,7 +347,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "TypeScript SDK e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -359,7 +359,7 @@ jobs:
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
workflow-name: "OpenLineage e2e test"
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -377,7 +377,7 @@ jobs:
compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }}
uses: ./.github/workflows/openlineage-e2e-compat-tests.yml
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
@@ -423,7 +423,7 @@ jobs:
airflow-ctl-integration-tests:
timeout-minutes: 60
name: "Airflow CTL integration tests with PROD image"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
.github/workflows/basic-tests.yml
+17 -9
View File
@@ -87,7 +87,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -105,7 +105,7 @@ jobs:
run-breeze-integration-tests:
timeout-minutes: 120
name: Breeze integration tests
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-breeze-integration-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -136,13 +136,15 @@ jobs:
# Without a pin uv picks the runner's system Python, which can be older than Airflow supports.
UV_PYTHON: "${{ inputs.default-python-version }}"
SHARED_DISTRIBUTIONS: ${{ join(fromJSON(inputs.shared-distributions-as-json), ' ') }}
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 1
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
@@ -171,7 +173,7 @@ jobs:
tests-scripts:
timeout-minutes: 10
name: Scripts tests
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-scripts-tests == 'true'
env:
# Without a pin uv picks the runner's system Python, which can be older than Airflow supports.
@@ -182,6 +184,8 @@ jobs:
with:
fetch-depth: 1
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
@@ -200,7 +204,7 @@ jobs:
tests-ui:
timeout-minutes: 15
name: React UI tests
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.run-ui-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -210,6 +214,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
@@ -237,7 +243,7 @@ jobs:
check-translation-completness:
timeout-minutes: 15
name: "Check translation completeness"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -254,7 +260,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -327,7 +333,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -381,7 +387,7 @@ jobs:
test-airflow-standalone:
timeout-minutes: 30
name: "Test Airflow standalone commands"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
AIRFLOW_HOME: ~/airflow
FORCE_COLOR: 1
@@ -390,6 +396,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Install uv"
# Extract uv version from uv.lock. The format is stable: the line
# immediately after `name = "uv"` is `version = "<X.Y.Z>"`.
.github/workflows/ci-amd.yml
+36 -25
View File
@@ -20,8 +20,9 @@
# `release-branch-ci.yml`. Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the
# scheduled canary. The two workflows are physical copies of each other
# (GitHub Actions has no cross-file YAML include), differing only in their
# trigger surface, the hardcoded `runner-type` / `platform` values in
# `build-info`'s outputs, and the architecture-specific print-platform job.
# trigger surface, the hardcoded `platform` value in `build-info`'s outputs
# (and the `PLATFORM` selective checks pick the runner type for), and the
# architecture-specific print-platform job.
# Edit both files together when changing pipeline shape.
name: Tests (AMD)
on: # yamllint disable-line rule:truthy
@@ -305,7 +306,7 @@ jobs:
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
# platform / runner-type are hardcoded per workflow file (this is the
# platform is hardcoded per workflow file (this is the
# AMD copy). The matching ARM copy lives in ci-arm.yml.
platform: "linux/amd64"
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
@@ -347,7 +348,7 @@ jobs:
run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }}
run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }}
runner-type: '["ubuntu-22.04"]'
runner-type: ${{ steps.selective-checks.outputs.runner-type }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }}
@@ -429,6 +430,8 @@ jobs:
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
# Repository variable: "true" moves every run, pushes included, back to public runners.
USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
PLATFORM: "linux/amd64"
SELECTIVE_TESTS_ON_PUSH: "true"
PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }}
@@ -566,7 +569,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
@@ -607,7 +610,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
@@ -729,7 +732,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -758,7 +761,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -787,7 +790,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -817,7 +820,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -846,7 +849,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -877,7 +880,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -908,7 +911,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -938,7 +941,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -1024,7 +1027,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1054,7 +1057,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1165,7 +1168,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1181,7 +1184,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1197,6 +1200,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
# keep this in sync with go.mod in go-sdk/
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
@@ -1235,7 +1240,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1245,6 +1250,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
@@ -1269,7 +1276,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1285,6 +1292,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup Java
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
with:
@@ -1299,7 +1308,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1331,7 +1340,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1368,7 +1377,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1389,7 +1398,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1669,7 +1678,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -1679,6 +1688,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
.github/workflows/ci-arm.yml
+37 -26
View File
@@ -20,9 +20,10 @@
# `release-branch-ci.yml` — never on PRs. Companion to `ci-amd.yml`, which runs the
# same pipeline on AMD for every PR. The two workflows are physical
# copies of each other (GitHub Actions has no cross-file YAML include),
# differing only in their trigger surface, the hardcoded `runner-type` /
# `platform` values in `build-info`'s outputs, and the architecture-specific
# print-platform job. Edit both files together when changing pipeline shape.
# differing only in their trigger surface, the hardcoded `platform` value in
# `build-info`'s outputs (and the `PLATFORM` selective checks pick the runner
# type for), and the architecture-specific print-platform job. Edit both files
# together when changing pipeline shape.
name: Tests (ARM)
on: # yamllint disable-line rule:truthy
schedule:
@@ -187,7 +188,7 @@ jobs:
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
# platform / runner-type are hardcoded per workflow file (this is the
# platform is hardcoded per workflow file (this is the
# ARM copy). The matching AMD copy lives in ci-amd.yml.
platform: "linux/arm64"
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
@@ -229,7 +230,7 @@ jobs:
run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }}
run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }}
runner-type: '["ubuntu-22.04-arm"]'
runner-type: ${{ steps.selective-checks.outputs.runner-type }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }}
@@ -311,6 +312,8 @@ jobs:
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
# Repository variable: "true" moves every run, pushes included, back to public runners.
USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}"
PLATFORM: "linux/arm64"
SELECTIVE_TESTS_ON_PUSH: "true"
PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }}
@@ -448,7 +451,7 @@ jobs:
timeout-minutes: 45
name: "MyPy providers checks"
needs: [build-info, build-ci-images]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }}
if: needs.build-info.outputs.run-mypy-providers == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
@@ -489,7 +492,7 @@ jobs:
timeout-minutes: 20
name: "Migration round-trip check"
needs: [build-info, build-ci-images]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
if: needs.build-info.outputs.has-migrations == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
@@ -611,7 +614,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -640,7 +643,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "postgres"
test-name: "Postgres"
@@ -669,7 +672,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -699,7 +702,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "mysql"
test-name: "MySQL"
@@ -728,7 +731,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -759,7 +762,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: "Sqlite"
@@ -790,7 +793,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -820,7 +823,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
backend: "sqlite"
test-name: ""
@@ -906,7 +909,7 @@ jobs:
if: >
needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -936,7 +939,7 @@ jobs:
packages: read
if: needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
test-name: "LowestDeps"
force-lowest-dependencies: "true"
@@ -1047,7 +1050,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1063,7 +1066,7 @@ jobs:
tests-go-sdk:
name: "Go SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1079,6 +1082,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
# keep this in sync with go.mod in go-sdk/
- name: Setup Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
@@ -1117,7 +1122,7 @@ jobs:
tests-ts-sdk:
name: "TypeScript SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1127,6 +1132,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
@@ -1151,7 +1158,7 @@ jobs:
tests-java-sdk:
name: "Java SDK tests"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }}
timeout-minutes: 15
permissions:
contents: read
@@ -1167,6 +1174,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: Setup Java
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
with:
@@ -1181,7 +1190,7 @@ jobs:
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1213,7 +1222,7 @@ jobs:
build-ts-sdk-docs:
name: "TypeScript SDK docs"
needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }}
timeout-minutes: 30
permissions:
contents: read
@@ -1250,7 +1259,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -1271,7 +1280,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ needs.build-info.outputs.runner-type }}
runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }}
platform: ${{ needs.build-info.outputs.platform }}
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
use-uv: ${{ needs.build-info.outputs.use-uv }}
@@ -1550,7 +1559,7 @@ jobs:
- tests-special
- tests-with-lowest-direct-resolution-core
- tests-with-lowest-direct-resolution-providers
runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }}
if: needs.build-info.outputs.run-unit-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -1560,6 +1569,8 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
.github/workflows/ci-image-build.yml
+24 -12
View File
@@ -128,7 +128,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 110
name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
@@ -147,24 +147,38 @@ jobs:
with:
ref: ${{ inputs.checkout-ref }}
persist-credentials: false
# A local action is read from the workspace, and `checkout-ref` may be a release ref cut
# before the stash actions existed, so they are loaded from the workflow's own version. The
# runner-preparing scripts too: an older `make_mnt_writeable.sh` empties /mnt, which on
# RunsOn holds the runner's own disks.
- name: "Checkout the workflow version for its actions and runner scripts"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
path: workflow-version
sparse-checkout: |
.github/actions
scripts/ci
- name: "Free up disk space"
shell: bash
run: ./scripts/tools/free_up_disk_space.sh
- name: "Make /mnt writeable"
run: ./scripts/ci/make_mnt_writeable.sh
run: ./workflow-version/scripts/ci/make_mnt_writeable.sh
- name: "Move docker to /mnt"
run: ./scripts/ci/move_docker_to_mnt.sh
run: ./workflow-version/scripts/ci/move_docker_to_mnt.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
# A ref is not a commit: a branch documented twice moves between the two publishes, and
# an image built before it moved is a cache, not an answer. The commit each stashed image
# is saved with tells the two apart, and it is stashed separately so that deciding costs
# a few bytes rather than the multi-gigabyte image the decision may make unnecessary.
# `only-current-branch` must stay true on the image restores below: runs on RunsOn (canaries
# and release-branch pushes) save these image stashes as S3 digests only, so a fallback to the
# base branch would find nothing on public runners anyway.
- name: >
Restore the commit the CI image stashed for ref ${{ inputs.image-stash-ref }}
was built from
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -197,8 +211,7 @@ jobs:
- name: >
Restore CI docker image built for ref ${{ inputs.image-stash-ref }}
${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -244,8 +257,7 @@ jobs:
--cache-file /tmp/ci-cache-mount-save-v4-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz
if: steps.restore-cache-mount.outputs.stash-hit == 'true'
- name: "Restore CI docker image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
# yamllint disable-line rule:line-length
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
uses: ./workflow-version/.github/actions/stash-restore
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
path: "/mnt/"
@@ -336,7 +348,7 @@ jobs:
shell: bash
if: inputs.upload-image-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true'
- name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
uses: ./workflow-version/.github/actions/stash-save
with:
key: ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
@@ -350,7 +362,7 @@ jobs:
# what a later publish of that same ref reads - hence a retention that outlives the shared
# one, the same ref being rebuilt three days apart at the very least.
- name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref }}"
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
uses: ./workflow-version/.github/actions/stash-save
with:
key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
@@ -364,7 +376,7 @@ jobs:
# Saved last and with the image's retention, so that finding this commit is enough to
# know the image it describes is there to be restored.
- name: "Stash the commit the CI image for ref ${{ inputs.image-stash-ref }} was built from"
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
uses: ./workflow-version/.github/actions/stash-save
with:
key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\
-${{ inputs.image-stash-ref }}"
.github/workflows/ci-image-checks.yml
+4 -4
View File
@@ -119,7 +119,7 @@ jobs:
static-checks:
timeout-minutes: 45
name: "Static checks"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}"
@@ -175,7 +175,7 @@ jobs:
build-docs:
timeout-minutes: 150
name: "Build documentation"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
if: inputs.docs-build == 'true'
env:
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -334,7 +334,7 @@ jobs:
id-token: write
contents: read
needs: build-docs
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -466,7 +466,7 @@ jobs:
test-python-api-client:
timeout-minutes: 60
name: "Test Python API client"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
if: inputs.run-api-codegen == 'true'
env:
BACKEND: "postgres"
.github/workflows/finalize-tests.yml
+4 -2
View File
@@ -80,7 +80,7 @@ permissions:
contents: read
jobs:
update-constraints:
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
timeout-minutes: 80
name: "Update constraints"
permissions:
@@ -103,6 +103,8 @@ jobs:
with:
# Needed to perform push action
persist-credentials: false
- name: "RunsOn cost summary"
uses: ./.github/actions/runs-on-cost-summary
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
@@ -124,7 +126,7 @@ jobs:
# Constraint commits are now handled by the dedicated update-constraints-on-push workflow
# triggered when uv.lock changes are pushed to main/test branches.
dependency-upgrade-summary:
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
needs: [update-constraints]
if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64'
name: "Deps ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
.github/workflows/generate-constraints.yml
+3 -1
View File
@@ -85,7 +85,7 @@ jobs:
contents: read
timeout-minutes: 70
name: Generate constraints for ${{ matrix.python-version }} on ${{ inputs.platform }}
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -114,6 +114,8 @@ jobs:
platform: ${{ inputs.platform }}
python: ${{ matrix.python-version }}
use-uv: ${{ inputs.use-uv }}
# Runs the checkout-ref's own make_mnt_writeable.sh, which on old refs empties /mnt and
# would kill a RunsOn runner: every caller passing checkout-ref pins public runners.
make-mnt-writeable-and-cleanup: true
- name: "Install prek"
uses: ./.github/actions/install-prek
.github/workflows/helm-tests.yml
+2 -2
View File
@@ -52,7 +52,7 @@ jobs:
name: >-
Unit tests Helm: ${{ matrix.helm-test-package }}
(K8S ${{ matrix.kubernetes-version }})
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -98,7 +98,7 @@ jobs:
tests-helm-release:
timeout-minutes: 80
name: "Release Helm"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/integration-system-tests.yml
+3 -3
View File
@@ -75,7 +75,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-core-integrations != '[]'
name: "Integration core ${{ matrix.integration }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -126,7 +126,7 @@ jobs:
timeout-minutes: 30
if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true'
name: "Integration: providers ${{ matrix.integration }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -176,7 +176,7 @@ jobs:
timeout-minutes: 30
if: inputs.run-system-tests == 'true'
name: "System Tests"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
env:
BACKEND: "postgres"
BACKEND_VERSION: ${{ inputs.default-postgres-version }}
.github/workflows/k8s-tests.yml
+2 -2
View File
@@ -62,7 +62,7 @@ jobs:
tests-kubernetes:
timeout-minutes: 60
name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
max-parallel: 20
@@ -149,7 +149,7 @@ jobs:
tests-kubernetes-lang-sdk:
timeout-minutes: 60
name: "K8S Lang-SDK:${{ inputs.lang-sdk-kubernetes-combo }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
env:
DEBUG_RESOURCES: ${{ inputs.debug-resources }}
INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }}
.github/workflows/kustomize-overlays-tests.yml
+1 -1
View File
@@ -53,7 +53,7 @@ jobs:
smoke-test-overlay:
timeout-minutes: 40
name: "Overlay smoke test: ${{ matrix.overlay }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }}
strategy:
fail-fast: false
max-parallel: 4
.github/workflows/prod-image-build.yml
+3 -3
View File
@@ -119,7 +119,7 @@ jobs:
build-prod-packages:
name: "Build Airflow and provider distributions"
timeout-minutes: 10
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
if: inputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -199,7 +199,7 @@ jobs:
python-version: ${{ fromJSON(inputs.python-versions) }}
timeout-minutes: 80
name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
needs:
- build-prod-packages
env:
@@ -289,7 +289,7 @@ jobs:
breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt"
if: inputs.upload-image-artifact == 'true'
- name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
uses: ./.github/actions/stash-save
with:
key: prod-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}
path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar"
.github/workflows/push-image-cache.yml
+2 -2
View File
@@ -79,7 +79,7 @@ on: # yamllint disable-line rule:truthy
jobs:
push-ci-image-cache:
name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
permissions:
contents: read
packages: write
@@ -149,7 +149,7 @@ jobs:
push-prod-image-cache:
name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
permissions:
contents: read
packages: write
.github/workflows/special-tests.yml
+11 -11
View File
@@ -94,7 +94,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -119,7 +119,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-sqlalchemy: "true"
test-name: "MinSQLAlchemy-Postgres"
@@ -144,7 +144,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -169,7 +169,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-sqlalchemy: "true"
test-name: "LatestSQLAlchemy-Postgres"
@@ -194,7 +194,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -220,7 +220,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
upgrade-boto: "true"
test-name: "LatestBoto-Postgres"
@@ -246,7 +246,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -272,7 +272,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 8, 32) }}
platform: ${{ inputs.platform }}
downgrade-pendulum: "true"
test-name: "Pendulum2-Postgres"
@@ -298,7 +298,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -323,7 +323,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 4, 16) }}
platform: ${{ inputs.platform }}
test-name: "Postgres"
test-scope: "Quarantined"
@@ -348,7 +348,7 @@ jobs:
contents: read
packages: read
with:
runners: ${{ inputs.runners }}
runners: ${{ format(inputs.runners, 32, 128) }}
platform: ${{ inputs.platform }}
test-name: "SystemTest"
test-scope: "System"
.github/workflows/test-providers.yml
+2 -2
View File
@@ -72,7 +72,7 @@ jobs:
prepare-install-verify-provider-distributions:
timeout-minutes: 80
name: "Providers ${{ matrix.package-format }} tests"
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
matrix:
@@ -178,7 +178,7 @@ jobs:
timeout-minutes: 80
# yamllint disable rule:line-length
name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.test-types.description }}
runs-on: ${{ fromJSON(inputs.runners) }}
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
strategy:
fail-fast: false
matrix:
.github/workflows/ui-e2e-tests.yml
+1 -1
View File
@@ -88,7 +88,7 @@ jobs:
test-ui-e2e-tests:
timeout-minutes: 90
name: ${{ inputs.workflow-name || 'UI E2E Tests' }}
runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }}
runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0], 8, 32) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11' }}"
GITHUB_REPOSITORY: ${{ github.repository }}
README.md
+2
View File
@@ -603,3 +603,5 @@ The CI infrastructure for Apache Airflow has been sponsored by:
<a href="https://astronomer.io"><img src="https://assets2.astronomer.io/logos/logoForLIGHTbackground.png" alt="astronomer.io" width="250px"></a>
<a href="https://aws.amazon.com/opensource/"><img src="https://github.com/apache/airflow/blob/main/providers/amazon/docs/integration-logos/AWS-Cloud-alt_light-bg@4x.png?raw=true" alt="AWS OpenSource" width="130px"></a>
The self-hosted CI runners on AWS are managed by [RunsOn](https://runs-on.com), which is free for open-source projects.
dev/breeze/doc/ci/04_selective_checks.md
+71 -15
View File
@@ -33,7 +33,7 @@
- [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks)
- [Suspended providers](#suspended-providers)
- [Selective check outputs](#selective-check-outputs)
- [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs)
- [Self-hosted (RunsOn) vs. public runners](#self-hosted-runson-vs-public-runners)
- [Changing behaviours of the CI runs by setting labels](#changing-behaviours-of-the-ci-runs-by-setting-labels)
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -689,6 +689,7 @@ GitHub Actions to pass the list of parameters to a command to execute
| run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | |
| amd-runners | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | |
| arm-runners | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | |
| runner-type | Runner labels for the CI jobs on this platform: public, or a RunsOn template jobs fill with cpu/ram | \["ubuntu-22.04"\] | |
| selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * |
| skip-prek-hooks | Which prek hooks should be skipped during the static-checks run | flynt,identity | |
| skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | |
@@ -698,27 +699,82 @@ GitHub Actions to pass the list of parameters to a command to execute
| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | |
## Committer vs. Non-committer PRs
## Self-hosted (RunsOn) vs. public runners
There is a difference in how the CI jobs are run for committer and non-committer PRs from forks.
The main reason is security; we do not want to run untrusted code on our infrastructure for self-hosted runners.
The `Tests (AMD)` / `Tests (ARM)` jobs run either on GitHub-hosted ("public") runners or on self-hosted
[RunsOn](https://runs-on.com) EC2 runners in the Airflow AWS account. Selective checks route runs of
code from `apache/airflow` itself there; this is a routing preference, not what keeps fork code off
RunsOn (see below). The `runner-type` output is decided as follows, first match wins:
Currently there is no difference because we are not using `self-hosted` runners (until we implement `Action
Runner Controller` but most of the jobs, committer builds will use "Self-hosted" runners by default,
while non-committer builds will use "Public" runners. For committers, this can be overridden by setting the
`use public runners` label in the PR.
1. Runs in any repository other than `apache/airflow` (forks running their own CI) use public runners.
2. Any run uses public runners while the repository variable `USE_PUBLIC_RUNNERS` is `true`: the
switch for moving `main` off RunsOn without a PR, e.g. when the stack is unavailable.
3. PRs with the `use public runners` label use public runners.
4. Any `push`, `schedule` or `workflow_dispatch` in `apache/airflow` (canary runs and the release
branch pushes, all triggered by people with write access) uses RunsOn.
5. Other events than `pull_request` use public runners.
6. PRs with the `use self-hosted runners` label use RunsOn, if a committer's (`COMMITTERS` in
`global_constants.py`) latest review of the PR's head commit is an approval and no committer's
request for changes stands. The label outlives the commit it was added for, so on its own it would
vouch for every later push: a new push sends the PR back to public runners until the head commit is
approved again, and so does a dismissed approval if no other committer's approval of the head commit
remains. A committer's request for changes keeps the PR on public runners across pushes, as it blocks
a merge on GitHub, until that committer reviews again or the review is dismissed. The label must
also still be on the PR: a re-run reuses the labels of its original event, so they are checked
against the PR's current ones. If the PR or its reviews cannot be read, the label is ignored as well.
7. PRs with the `non committer build` label use public runners.
8. PRs opened by a bot (Dependabot and the like), or from a `dependabot/` branch, use public runners:
they carry code nobody has reviewed yet.
9. PRs from a branch of `apache/airflow` use RunsOn; PRs from forks use public runners.
Labels are read from the event that started the run, and the CI workflows do not run on `labeled`. After
adding or removing a label, push a commit or close and reopen the PR; re-running an earlier run reuses
its original labels, and "Re-run failed jobs" also reuses that run's runner decision. A push voids the
approval `use self-hosted runners` needs, so for that label approve the head commit and then close and
reopen the PR.
The RunsOn value is a label template such as
`["runs-on=<run_id>/family=.../image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]`. Every job
picks its size class with `format(fromJSON(inputs.runners)[0], <cpu>, <ram>)` (4/16, 8/32 or 32/128),
or, for a single-job reusable workflow, its caller passes `format(<runners>, <cpu>, <ram>)`;
on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are
gated on `runner.environment == 'self-hosted'`:
* The EC2 cost summary of each job.
* The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to
the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact.
Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by
the same run, branch and repository (or, for a PR, its base branch). The smaller caches (mount cache, prek, docs inventory) use
the ASF stash action on both kinds of runners, so PRs on public runners keep the ones canary runs save;
the price is that every RunsOn canary uploads the mount cache from AWS to GitHub.
The decision above is not a security boundary. Under the `pull_request` event the code that
evaluates it is the PR's own: a fork PR runs its own workflow files and breeze sources, and can name
RunsOn labels in `runs-on:` directly. RunsOn launches a runner for any job of a repository its GitHub
App is installed on, so nothing in this repository can prevent that. Keeping untrusted code away from
anything that matters has to be enforced outside the repository, by:
* The repository's Actions setting "Approval for running fork pull request workflows from
contributors", which must be "Require approval for all external contributors". That still exempts
members of the `apache` GitHub organization (ASF committers of any project who linked their GitHub
account) and repository collaborators, but every exempt person is identifiable.
* The RunsOn stack, which must be dedicated to this repository and give the runners an instance role
that reaches nothing beyond the stack's own cache bucket (whose contents are never trusted, see the
stash actions), no EBS permissions, no SSH, network isolation between runners and a maximum run
time. RunsOn has no cap on instance sizes or counts and its budget only alerts, so the AWS account's
EC2 vCPU quotas are the cost ceiling.
Pushes in a contributor fork are never committer builds, whoever pushes, because forks have no
self-hosted runners.
## Changing behaviours of the CI runs by setting labels
Also, currently for most of the jobs, committer builds by default use "Self-hosted" runners, while
non-committer builds use "Public" runners. For committers, this can be overridden by setting the
`use public runners` label in the PR. In the future when we might also switch committers to public runners.
Committers will be able to use `use self-hosted runners` label in the PR to force using self-hosted runners.
The `use public runners` label will still be available for committers and they will be able to set it for
builds that also have `canary` label set to also switch the `canary` builds to public runners.
PRs from `apache/airflow` branches run on self-hosted RunsOn runners and PRs from forks run on public
runners (see above). Set `use public runners` to move a PR to public runners; a committer can set
`use self-hosted runners` and approve the PR's head commit to move a fork PR to RunsOn, provided no
committer's request for changes stands, then close and reopen the PR (a push would need a fresh
approval; a committer should do the reopen, since an external author's reopen still needs a maintainer
to approve the workflow run).
If you are testing CI workflow changes and want to test it for more complete matrix combinations generated by
the jobs - you can set `all versions` label in the PR. This will run the PRs with the same combinations
@@ -726,7 +782,7 @@ of versions as the `canary` main build. Using `all versions` is automatically se
change in `pyproject.toml`.
If you are running an `apache` PR, you can also set `canary` label for such PR and in this case, all the
`canary` properties of build will be used: `self-hosted` runners, `full tests needed` mode, `all versions`
`canary` properties of build will be used: `full tests needed` mode, `all versions`
as well as all canary-specific jobs will run there. You can modify this behaviour of the `canary` run by
applying `use public runners`, and `default versions only` labels to the PR as well which will still run
a `canary` equivalent build but with public runners an default Python/K8S versions only - respectively.
dev/breeze/doc/ci/06_debugging.md
+5 -1
View File
@@ -44,12 +44,16 @@ in some cases, you need to run the PR as coming from the "apache"
repository rather than from your fork.
You can also apply the label later and rebase the PR or close/reopen
the PR to apply the label to the PR.
the PR to apply the label to the PR. For `use self-hosted runners`, a
committer approves the head commit (with no committer's request for
changes standing) and closes/reopens the PR: a rebase needs a fresh
approval.
| Action to perform | Label to set | PR from "apache" repo |
|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|:---------------------:|
| Run the build with all combinations of all<br>python, backends, kubernetes etc on PR, <br>and run all types of tests for all test<br>groups. | full tests needed | |
| Force to use public runners for the build | use public runners | |
| Run a PR from a fork on the self-hosted RunsOn<br>runners (needs a committer's approval of its head commit<br>and no committer's request for changes) | use self-hosted runners | |
| Debug resources used during the build for <br>parallel jobs | debug ci resources | |
| Force running PR on latest versions of<br>python, backends, kubernetes etc. when you<br>want to save resources and test only latest<br>versions | latest versions only | |
| Force running PR on minimal (default) <br>versions of python, backends, kubernetes etc.<br>in order to save resources and run tests only<br>for minimum versions | default versions only | |
dev/breeze/src/airflow_breeze/global_constants.py
+8
View File
@@ -38,6 +38,14 @@ from airflow_breeze.utils.path_utils import (
PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]'
PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]'
# RunsOn (runs-on.com) EC2 runner label templates. Jobs fill in their size class with
# `format(<label>, cpu, ram)` in the workflow; the same call leaves the public labels unchanged.
RUNS_ON_AMD_RUNNER_TEMPLATE = (
"runs-on={run_id}/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={{0}}/ram={{1}}"
)
RUNS_ON_ARM_RUNNER_TEMPLATE = (
"runs-on={run_id}/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={{0}}/ram={{1}}"
)
# Platform the tests of a CI run execute on.
CI_AMD_PLATFORM = "linux/amd64"
dev/breeze/src/airflow_breeze/utils/selective_checks.py
+116
View File
@@ -55,6 +55,8 @@ from airflow_breeze.global_constants import (
PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
PUBLIC_AMD_RUNNERS,
PUBLIC_ARM_RUNNERS,
RUNS_ON_AMD_RUNNER_TEMPLATE,
RUNS_ON_ARM_RUNNER_TEMPLATE,
TESTABLE_CORE_INTEGRATIONS,
TESTABLE_PROVIDERS_INTEGRATION_OWNERS,
TESTABLE_PROVIDERS_INTEGRATIONS,
@@ -95,6 +97,7 @@ LATEST_VERSIONS_ONLY_LABEL = "latest versions only"
NON_COMMITTER_BUILD_LABEL = "non committer build"
UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies"
USE_PUBLIC_RUNNERS_LABEL = "use public runners"
USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners"
ALLOW_PROVIDER_DEPENDENCY_BUMP_LABEL = "allow provider dependency bump"
SKIP_COMMON_COMPAT_CHECK_LABEL = "skip common compat check"
AREA_E2E_TESTS_LABEL = "area:e2e-tests"
@@ -2179,6 +2182,119 @@ class SelectiveChecks:
def arm_runners(self) -> str:
return PUBLIC_ARM_RUNNERS
@cached_property
def runner_type(self) -> str:
"""Runner labels for the jobs of a CI run on this run's platform.
The RunsOn label is a template the jobs fill in with their size class.
"""
# RunsOn needs the run id in the label; without one (e.g. a local run) the jobs would never start.
run_id = self._github_context_dict.get("run_id")
if not run_id or not self._use_runs_on_runners():
return PUBLIC_ARM_RUNNERS if self._platform == CI_ARM_PLATFORM else PUBLIC_AMD_RUNNERS
template = (
RUNS_ON_ARM_RUNNER_TEMPLATE if self._platform == CI_ARM_PLATFORM else RUNS_ON_AMD_RUNNER_TEMPLATE
)
return json.dumps([template.format(run_id=run_id)])
def _use_runs_on_runners(self) -> bool:
"""Whether this run should use the self-hosted RunsOn runners.
This only routes runs; it does not keep fork code off RunsOn. A ``pull_request`` run uses
the PR's own workflow files and breeze, so a fork can request a RunsOn label directly -
that restriction has to be enforced by the RunsOn stack and the repository's Actions
settings, not here. Routed there: pushes, schedules and dispatches, and PRs from
apache/airflow branches except bots' (e.g. Dependabot's). A fork PR goes there while it
carries the ``use self-hosted runners`` label, a committer has approved its head commit and
no committer's request for changes is outstanding.
The repository variable ``USE_PUBLIC_RUNNERS=true`` moves every run back to public runners.
"""
if self._github_repository != APACHE_AIRFLOW_GITHUB_REPOSITORY:
return False
if os.environ.get("USE_PUBLIC_RUNNERS", "").casefold() == "true":
return False
if USE_PUBLIC_RUNNERS_LABEL in self._pr_labels:
return False
if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE, GithubEvents.WORKFLOW_DISPATCH]:
return True
if self._github_event != GithubEvents.PULL_REQUEST:
return False
if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels and self._is_self_hosted_label_approved():
return True
if NON_COMMITTER_BUILD_LABEL in self._pr_labels:
return False
pull_request = self._github_context_dict.get("event", {}).get("pull_request", {})
# The PR author, not github.actor: a committer rebasing a Dependabot PR becomes the actor.
# The branch too: anyone can open their own PR from a Dependabot branch.
if (pull_request.get("user") or {}).get("type") == "Bot" or (
pull_request.get("head", {}).get("ref", "").startswith("dependabot/")
):
return False
# A PR from a deleted fork has no head repo.
head_repo = (pull_request.get("head", {}).get("repo") or {}).get("full_name")
return head_repo == APACHE_AIRFLOW_GITHUB_REPOSITORY
def _is_self_hosted_label_approved(self) -> bool:
"""Whether the PR still has the label, a committer approved its head commit and no committer's veto stands.
A re-run reuses the labels of its original event, so a later removal only shows in the PR
itself. The label outlives the commit it was added for, so on its own it would vouch for
every later push. A request for changes stays in force across pushes, as on GitHub, until
that committer reviews again or it is dismissed. Any failure to tell counts as "no".
"""
# Lazy: breeze keeps its top-level imports minimal (check-breeze-top-dependencies-limited).
import requests
pull_request = self._github_context_dict.get("event", {}).get("pull_request", {})
head_sha = pull_request.get("head", {}).get("sha")
if not head_sha:
return False
pull_request_url = (
f"https://api.github.com/repos/{self._github_repository}/pulls/{pull_request.get('number')}"
)
headers = {"Accept": "application/vnd.github+json"}
if token := os.environ.get("GITHUB_TOKEN"):
headers["Authorization"] = f"Bearer {token}"
# Each reviewer's latest verdict, on the head commit and on any commit; a dismissed review is
# listed as DISMISSED in place, and comment-only reviews change nothing.
latest_on_head: dict[str, str] = {}
latest_overall: dict[str, str] = {}
try:
response = requests.get(pull_request_url, headers=headers, timeout=30)
response.raise_for_status()
if USE_SELF_HOSTED_RUNNERS_LABEL not in {label["name"] for label in response.json()["labels"]}:
console_print(
f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r} label: the PR no longer has it"
)
return False
url: str | None = f"{pull_request_url}/reviews?per_page=100"
while url:
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
for review in response.json():
if review.get("state") not in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
continue
login = ((review.get("user") or {}).get("login") or "").casefold()
latest_overall[login] = review["state"]
if review.get("commit_id") == head_sha:
latest_on_head[login] = review["state"]
url = response.links.get("next", {}).get("url")
# AttributeError / KeyError / TypeError: a 200 whose body is not the expected PR or list of reviews.
except (requests.RequestException, AttributeError, KeyError, TypeError) as e:
console_print(f"[warning]Could not read the labels and reviews of the PR: {e}")
return False
committers = {committer.casefold() for committer in COMMITTERS}
approved = any(latest_on_head.get(committer) == "APPROVED" for committer in committers)
vetoed = any(latest_overall.get(committer) == "CHANGES_REQUESTED" for committer in committers)
if approved and not vetoed:
return True
console_print(
f"[warning]Ignoring the {USE_SELF_HOSTED_RUNNERS_LABEL!r} label: "
f"the head commit {head_sha} is not approved by a committer, or a committer's request for "
"changes stands"
)
return False
@cached_property
def has_migrations(self) -> bool:
return any([file.startswith("airflow-core/src/airflow/migrations/") for file in self._files])
dev/breeze/tests/test_selective_checks.py
+411
View File
@@ -23,6 +23,7 @@ from typing import Any
from unittest.mock import Mock, patch
import pytest
import requests
from rich.console import Console
from airflow_breeze.global_constants import (
@@ -30,6 +31,7 @@ from airflow_breeze.global_constants import (
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
CI_AMD_PLATFORM,
CI_ARM_PLATFORM,
COMMITTERS,
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS,
DEFAULT_KUBERNETES_VERSION,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
@@ -4905,3 +4907,412 @@ def test_providers_with_test_side_effects_are_shared_process_providers():
for provider in PROVIDERS_WITH_PROCESS_GLOBAL_TEST_SIDE_EFFECTS:
assert provider in get_available_distributions()
assert provider not in LONG_RUNNING_TEST_PROVIDERS
RUNS_ON_AMD = '["runs-on=42/family=m7a+m7i+m6a+m6i/image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]'
RUNS_ON_ARM = '["runs-on=42/family=m8g+m7g+m6g/image=ubuntu22-full-arm64/volume=150gb/cpu={0}/ram={1}"]'
HEAD_SHA = "a" * 40
@pytest.fixture(autouse=True)
def _no_use_public_runners_variable(monkeypatch):
monkeypatch.delenv("USE_PUBLIC_RUNNERS", raising=False)
def _pull_request_context(head_repo: str | None, author_type: str = "User") -> dict[str, Any]:
head = {"repo": {"full_name": head_repo} if head_repo else None, "sha": HEAD_SHA}
return {
"run_id": 42,
"event": {"pull_request": {"number": 7, "head": head, "user": {"type": author_type}}},
}
@pytest.mark.parametrize(
("github_event", "github_repository", "github_context_dict", "pr_labels", "platform", "expected"),
[
pytest.param(
GithubEvents.PUSH,
"apache/airflow",
{"run_id": 42},
(),
CI_AMD_PLATFORM,
RUNS_ON_AMD,
id="canary push uses RunsOn",
),
pytest.param(
GithubEvents.SCHEDULE,
"apache/airflow",
{"run_id": 42},
(),
CI_ARM_PLATFORM,
RUNS_ON_ARM,
id="scheduled ARM canary uses ARM RunsOn",
),
pytest.param(
GithubEvents.WORKFLOW_DISPATCH,
"apache/airflow",
{"run_id": 42},
(),
CI_AMD_PLATFORM,
RUNS_ON_AMD,
id="workflow_dispatch uses RunsOn",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("apache/airflow", author_type="Bot"),
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="bot PR from an apache/airflow branch uses public runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
{
"run_id": 42,
"event": {
"pull_request": {
"head": {"ref": "dependabot/uv/foo", "repo": {"full_name": "apache/airflow"}},
"user": {"type": "User"},
}
},
},
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="user PR from a Dependabot branch uses public runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("apache/airflow", author_type="Bot"),
("use self-hosted runners", "non committer build"),
CI_AMD_PLATFORM,
RUNS_ON_AMD,
id="committer's self-hosted label wins over bot author and non committer build",
),
pytest.param(
GithubEvents.PUSH,
"apache/airflow",
{},
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="push without a run id stays on public runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("someone/airflow"),
("use self-hosted runners", "use public runners"),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="public runners label wins over the self-hosted label",
),
pytest.param(
GithubEvents.PUSH,
"someone/airflow",
{"run_id": 42},
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="push in a fork repository stays on public runners",
),
pytest.param(
GithubEvents.SCHEDULE,
"someone/airflow",
{"run_id": 42},
(),
CI_ARM_PLATFORM,
'["ubuntu-22.04-arm"]',
id="schedule in a fork repository stays on public ARM runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("apache/airflow"),
(),
CI_AMD_PLATFORM,
RUNS_ON_AMD,
id="PR from an apache/airflow branch uses RunsOn",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("someone/airflow"),
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="PR from a fork uses public runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context(None),
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="PR from a deleted fork uses public runners",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("someone/airflow"),
("use self-hosted runners",),
CI_AMD_PLATFORM,
RUNS_ON_AMD,
id="PR from a fork with the self-hosted label uses RunsOn",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("someone/airflow"),
("canary", "full tests needed"),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="canary label does not route a fork PR to RunsOn",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("apache/airflow"),
("use public runners",),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="public runners label opts an apache/airflow PR out",
),
pytest.param(
GithubEvents.PULL_REQUEST,
"apache/airflow",
_pull_request_context("apache/airflow"),
("non committer build",),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="non committer build label treats an apache/airflow PR as a fork PR",
),
pytest.param(
GithubEvents.PULL_REQUEST_TARGET,
"apache/airflow",
_pull_request_context("someone/airflow"),
(),
CI_AMD_PLATFORM,
'["ubuntu-22.04"]',
id="pull_request_target uses public runners",
),
],
)
@patch.object(SelectiveChecks, "_is_self_hosted_label_approved", autospec=True, return_value=True)
def test_runner_type(
_mock_is_self_hosted_label_approved,
github_event: GithubEvents,
github_repository: str,
github_context_dict: dict[str, Any],
pr_labels: tuple[str, ...],
platform: str,
expected: str,
):
selective_checks = SelectiveChecks(
github_event=github_event,
github_repository=github_repository,
github_context_dict=github_context_dict,
pr_labels=pr_labels,
platform=platform,
)
assert selective_checks.runner_type == expected
@pytest.mark.parametrize(
("value", "expected"),
[
pytest.param("true", '["ubuntu-22.04"]', id="true moves a push to public runners"),
pytest.param("True", '["ubuntu-22.04"]', id="case does not matter"),
pytest.param("false", RUNS_ON_AMD, id="false keeps RunsOn"),
pytest.param("", RUNS_ON_AMD, id="an unset variable keeps RunsOn"),
],
)
def test_use_public_runners_variable(monkeypatch, value: str, expected: str):
monkeypatch.setenv("USE_PUBLIC_RUNNERS", value)
selective_checks = SelectiveChecks(
github_event=GithubEvents.PUSH, github_repository="apache/airflow", github_context_dict={"run_id": 42}
)
assert selective_checks.runner_type == expected
@patch.object(SelectiveChecks, "_is_self_hosted_label_approved", autospec=True, return_value=True)
def test_use_public_runners_variable_beats_the_self_hosted_label(
_mock_is_self_hosted_label_approved, monkeypatch
):
monkeypatch.setenv("USE_PUBLIC_RUNNERS", "true")
selective_checks = SelectiveChecks(
github_event=GithubEvents.PULL_REQUEST,
github_repository="apache/airflow",
github_context_dict=_pull_request_context("apache/airflow"),
pr_labels=("use self-hosted runners",),
)
assert selective_checks.runner_type == '["ubuntu-22.04"]'
def _review(login: str, state: str, commit_id: str = HEAD_SHA) -> dict[str, Any]:
return {"user": {"login": login}, "state": state, "commit_id": commit_id}
@pytest.mark.parametrize(
("pages", "expected"),
[
pytest.param([[_review(COMMITTERS[0], "APPROVED")]], True, id="approved by a committer"),
pytest.param(
[[_review(COMMITTERS[0].swapcase(), "APPROVED")]],
True,
id="approved by a committer whose login differs in case",
),
pytest.param([[_review("a-triager", "APPROVED")]], False, id="approved by a non-committer"),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED", commit_id="b" * 40)]],
False,
id="approved an earlier commit only",
),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "CHANGES_REQUESTED")]],
False,
id="changes requested after approving",
),
pytest.param(
[[_review(COMMITTERS[0], "CHANGES_REQUESTED"), _review(COMMITTERS[0], "APPROVED")]],
True,
id="approved after requesting changes",
),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "DISMISSED")]],
False,
id="approval dismissed",
),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[0], "COMMENTED")]],
True,
id="a later comment keeps the approval",
),
pytest.param(
[[_review("a-triager", "APPROVED")], [_review(COMMITTERS[0], "APPROVED")]],
True,
id="approved on a later page",
),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED"), _review(COMMITTERS[1], "CHANGES_REQUESTED")]],
False,
id="another committer's request for changes vetoes an approval",
),
pytest.param(
[[_review(COMMITTERS[0], "APPROVED"), _review("a-triager", "CHANGES_REQUESTED")]],
True,
id="a non-committer's request for changes does not veto",
),
pytest.param(
[
[
_review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b" * 40),
_review(COMMITTERS[0], "APPROVED"),
]
],
False,
id="a committer's request for changes on an earlier commit still vetoes",
),
pytest.param(
[
[
_review(COMMITTERS[1], "CHANGES_REQUESTED", commit_id="b" * 40),
_review(COMMITTERS[1], "APPROVED", commit_id="b" * 40),
_review(COMMITTERS[0], "APPROVED"),
]
],
True,
id="a request for changes withdrawn by its committer's later approval does not veto",
),
pytest.param([[]], False, id="no review"),
pytest.param([{"message": "Not Found"}], False, id="error object instead of reviews"),
],
)
@patch("requests.get", autospec=True)
def test_is_self_hosted_label_approved(mock_get, pages: list[list[dict[str, Any]]], expected: bool):
pull_request = Mock(spec=requests.Response)
pull_request.json.return_value = {"labels": [{"name": "use self-hosted runners"}]}
responses = [pull_request]
for index, page in enumerate(pages):
response = Mock(spec=requests.Response)
response.json.return_value = page
response.links = {"next": {"url": f"page-{index + 1}"}} if index + 1 < len(pages) else {}
responses.append(response)
mock_get.side_effect = responses
selective_checks = SelectiveChecks(
github_context_dict=_pull_request_context("someone/airflow"),
pr_labels=("use self-hosted runners",),
)
assert selective_checks._is_self_hosted_label_approved() is expected
assert [call.args[0] for call in mock_get.call_args_list[:2]] == [
"https://api.github.com/repos/apache/airflow/pulls/7",
"https://api.github.com/repos/apache/airflow/pulls/7/reviews?per_page=100",
]
@pytest.mark.parametrize(
"pull_request_body",
[
pytest.param({"labels": [{"name": "full tests needed"}]}, id="label removed since the event"),
pytest.param({"message": "Not Found"}, id="error object instead of the PR"),
],
)
@patch("requests.get", autospec=True)
def test_is_self_hosted_label_approved_needs_the_current_label(mock_get, pull_request_body: dict[str, Any]):
# A re-run reuses the labels of its original event, so the label is still in pr_labels.
response = Mock(spec=requests.Response)
response.json.return_value = pull_request_body
mock_get.return_value = response
selective_checks = SelectiveChecks(
github_context_dict=_pull_request_context("someone/airflow"),
pr_labels=("use self-hosted runners",),
)
assert selective_checks._is_self_hosted_label_approved() is False
mock_get.assert_called_once()
@patch("requests.get", autospec=True)
def test_is_self_hosted_label_approved_without_head_sha(mock_get):
# A review whose commit_id is missing must not match a missing head sha.
response = Mock(spec=requests.Response)
response.json.return_value = [{"user": {"login": COMMITTERS[0]}, "state": "APPROVED"}]
response.links = {}
mock_get.return_value = response
context = _pull_request_context("someone/airflow")
del context["event"]["pull_request"]["head"]["sha"]
selective_checks = SelectiveChecks(github_context_dict=context, pr_labels=("use self-hosted runners",))
assert selective_checks._is_self_hosted_label_approved() is False
mock_get.assert_not_called()
@pytest.mark.parametrize(
("head_repo", "expected"),
[
pytest.param("someone/airflow", '["ubuntu-22.04"]', id="fork PR falls back to public runners"),
pytest.param("apache/airflow", RUNS_ON_AMD, id="apache/airflow PR stays on RunsOn"),
],
)
@pytest.mark.parametrize(
"failure",
[
pytest.param(requests.ConnectionError("down"), id="connection error"),
pytest.param(requests.HTTPError("403"), id="non-2xx response"),
],
)
@patch("requests.get", autospec=True)
def test_self_hosted_label_ignored_when_reviews_cannot_be_read(
mock_get, failure: Exception, head_repo: str, expected: str
):
mock_get.side_effect = failure
selective_checks = SelectiveChecks(
github_event=GithubEvents.PULL_REQUEST,
github_context_dict=_pull_request_context(head_repo),
pr_labels=("use self-hosted runners",),
)
assert selective_checks.runner_type == expected
scripts/ci/make_mnt_writeable.sh
+17 -4
View File
@@ -22,13 +22,26 @@ function make_mnt_writeable {
sudo blkid
echo "Checking free space!"
df -H
echo "Cleaning /mnt just in case it is not empty"
echo "Cleaning /mnt just in case it is not empty and making sure that /mnt is writeable"
sudo mkdir -p /mnt
sudo rm -rf /mnt/*
if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then
# RunsOn mounts an instance's local NVMe disks under /mnt and keeps docker, /home/runner and
# /tmp on them, so emptying or re-owning such a mount would break the running job.
local entry
for entry in /mnt/*; do
if mountpoint -q "${entry}"; then
echo "Keeping mount point ${entry}"
else
sudo rm -rf --one-file-system "${entry}"
fi
done
sudo chown "${USER}" /mnt
else
sudo rm -rf /mnt/*
sudo chown -R "${USER}" /mnt
fi
echo "Checking free space!"
df -H
echo "Making sure that /mnt is writeable"
sudo chown -R "${USER}" /mnt
}
make_mnt_writeable
scripts/ci/move_docker_to_mnt.sh
+6 -1
View File
@@ -29,7 +29,12 @@ function cleanup_runner {
# Hence we only move docker to /mnt on x86_64 where we have a separate /mnt mount
# If we get short on disk space on arm64 as well we need to revisit this logic
# and make the idle nvme being used as well.
if uname -i|grep -q x86_64; then
if [[ "${RUNNER_ENVIRONMENT:-}" == "self-hosted" ]]; then
# RunsOn keeps docker either on the instance's local NVMe disks, which may also back
# /home/runner and /tmp, or - on instance types without them - on the same EBS root
# volume /mnt is on. Moving it gains nothing and could delete the running job's files.
echo "Self-hosted runner - keeping /var/lib/docker where it is"
elif uname -i|grep -q x86_64; then
local target_docker_volume_location="/mnt/var-lib-docker"
# This is faster than docker prune
echo "Stopping docker"
scripts/ci/prek/check_ci_workflows_in_sync.py
+2 -3
View File
@@ -37,8 +37,8 @@ Documented divergences:
its ``gate`` job runs selective tests only for pushes to the branches
in CANONICAL_PUSH_BRANCH_FILTERS (fork_ci_gate.py).
4. ``concurrency.group`` prefix — ``ci-arm-`` vs ``ci-amd-``
5. ``build-info`` outputs ``platform`` and ``runner-type`` — hardcoded per
architecture (and the surrounding comment naming the "ARM/AMD copy")
5. ``build-info`` output ``platform`` — hardcoded per architecture (and the
surrounding comment naming the "ARM/AMD copy")
6. ``PLATFORM`` env on the ``selective-checks`` step — the platform this
workflow's tests run on.
7. ``print-platform`` job — ``name:`` and the architecture echoed to
@@ -96,7 +96,6 @@ LINE_RULES: list[tuple[str, str]] = [
" ci-PLACEHOLDER-${{ github.event.pull_request.number || github.ref }}${{",
),
(r'^ platform: "linux/(?:arm64|amd64)"$', ' platform: "linux/PLACEHOLDER"'),
(r"^ runner-type: '\[\"ubuntu-22\.04(?:-arm)?\"\]'$", " runner-type: 'PLACEHOLDER'"),
(r'^ PLATFORM: "linux/(?:arm64|amd64)"$', ' PLATFORM: "linux/PLACEHOLDER"'),
(
r"^ # (?:ARM|AMD) copy\)\. The matching (?:AMD|ARM) copy lives in ci-(?:amd|arm)\.yml\.$",