diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index cb49d1d6d57..9fe5b551096 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -29,6 +29,8 @@ outputs: runs: using: "composite" steps: + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Setup python" uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6c91e28bff8..263db0346d4 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -82,12 +82,13 @@ runs: - name: > Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }} ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }} - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + uses: ./.github/actions/stash-restore with: key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\ ${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}" path: "/mnt/" + # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image + # stashes as S3 digests only. only-current-branch: 'true' fail-on-download: 'true' - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/actions/prepare_single_ci_image/action.yml b/.github/actions/prepare_single_ci_image/action.yml index d70012c80f7..8a734ef1102 100644 --- a/.github/actions/prepare_single_ci_image/action.yml +++ b/.github/actions/prepare_single_ci_image/action.yml @@ -36,11 +36,12 @@ runs: using: "composite" steps: - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}" - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + uses: ./.github/actions/stash-restore with: key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }} path: "/mnt/" + # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image + # stashes as S3 digests only. only-current-branch: 'true' fail-on-download: 'true' if: contains(inputs.python-versions-list-as-string, inputs.python) diff --git a/.github/actions/runs-on-cost-summary/action.yml b/.github/actions/runs-on-cost-summary/action.yml new file mode 100644 index 00000000000..fd0d2007f10 --- /dev/null +++ b/.github/actions/runs-on-cost-summary/action.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job +# that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze. +name: 'RunsOn cost summary' +description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.' +runs: + using: 'composite' + steps: + - name: "RunsOn cost summary" + if: runner.environment == 'self-hosted' + uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + with: + show_costs: summary diff --git a/.github/actions/stash-restore/action.yml b/.github/actions/stash-restore/action.yml new file mode 100644 index 00000000000..ed9535c1722 --- /dev/null +++ b/.github/actions/stash-restore/action.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On RunsOn (the only +# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's in-region S3 cache +# bucket instead of leaving AWS as GitHub artifacts. +# +# Every RunsOn job can write anywhere in that bucket, so the bucket is not trusted: objects are +# named by their sha256, and the digest itself travels as a small ASF stash artifact, which +# GitHub scopes to the run / branch exactly like a regular stash. A blob that does not match the +# digest the ASF restore found for this run, branch and repository is never extracted. The bucket's +# lifecycle rule (S3CacheExpirationInDays) must outlive the longest retention-days of a digest +# (6 days today), or a digest hit finds no blob. +name: 'Restore stash' +description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn runners.' +inputs: + key: + description: 'Stash key.' + required: true + path: + description: > + Directory to restore into (GitHub-hosted only; the S3 tarball restores entries to the + paths they were saved from, and the restore then fails if any of them is outside this directory). + required: false + default: ${{ github.workspace }} + only-current-branch: + description: 'Do not fall back to the base branch stash.' + required: false + default: 'false' + fail-on-download: + description: 'Fail when a stash was found but could not be restored.' + required: false + default: 'false' +outputs: + stash-hit: + description: "'true' if a stash was restored, 'false' otherwise." + value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit }} +runs: + using: 'composite' + steps: + - name: "Restore stash from GitHub artifacts" + id: artifact + if: runner.environment != 'self-hosted' + # yamllint disable-line rule:line-length + uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + with: + key: ${{ inputs.key }} + path: ${{ inputs.path }} + only-current-branch: ${{ inputs.only-current-branch }} + fail-on-download: ${{ inputs.fail-on-download }} + - name: "Restore stash digest from GitHub artifacts" + id: digest + if: runner.environment == 'self-hosted' + # yamllint disable-line rule:line-length + uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + with: + key: ${{ inputs.key }}-s3 + path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }} + only-current-branch: ${{ inputs.only-current-branch }} + fail-on-download: ${{ inputs.fail-on-download }} + - name: "Restore stash from S3" + id: s3 + if: runner.environment == 'self-hosted' + shell: bash + env: + STASH_KEY: ${{ inputs.key }} + STASH_PATH: ${{ inputs.path }} + DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }} + DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}/sha256 + FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }} + run: | + set -uo pipefail + payload="${RUNNER_TEMP}/stash-restore-$$.tar" + entries="${RUNNER_TEMP}/stash-restore-$$.entries" + miss() { + rm -f "${payload}" "${entries}" + echo "stash-hit=false" >> "${GITHUB_OUTPUT}" + if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true" ]]; then + echo "::error::$1" + exit 1 + fi + echo "$1" + exit 0 + } + if [[ "${DIGEST_HIT}" != "true" ]]; then + miss "No stash '${STASH_KEY}'" + fi + digest="$(cat "${DIGEST_FILE}")" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + miss "Malformed digest for stash '${STASH_KEY}'" + fi + bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}" + export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}" + object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar" + if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then + miss "Could not download ${object} for stash '${STASH_KEY}'" + fi + if ! echo "${digest} ${payload}" | sha256sum --check --status; then + miss "${object} does not match the digest recorded for stash '${STASH_KEY}'" + fi + # GNU tar detects the compression stash-save chose. + if ! tar -xPvf "${payload}" > "${entries}"; then + miss "Could not extract ${object} for stash '${STASH_KEY}'" + fi + rm -f "${payload}" + # The S3 tarball ignores `path`: fail loudly if a caller saves and restores different + # directories, which would work on GitHub-hosted runners only. + restore_dir="$(realpath -ms "${STASH_PATH}")" + restore_dir="${restore_dir%/}" + outside=() + while IFS= read -r entry; do + [[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}" + [[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}") + done < "${entries}" + rm -f "${entries}" + if [[ ${#outside[@]} -gt 0 ]]; then + printf '%s\n' "${outside[@]}" + echo "::error::Stash '${STASH_KEY}' restored the entries above outside of ${restore_dir}/" + exit 1 + fi + echo "stash-hit=true" >> "${GITHUB_OUTPUT}" + echo "Restored stash '${STASH_KEY}' from ${object}" diff --git a/.github/actions/stash-save/action.yml b/.github/actions/stash-save/action.yml new file mode 100644 index 00000000000..ecfd2409438 --- /dev/null +++ b/.github/actions/stash-save/action.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn the tarball goes +# to the stack's S3 cache bucket under its sha256 and only the digest is saved as an ASF stash +# artifact. See stash-restore for why. +name: 'Save stash' +description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.' +inputs: + key: + description: 'Stash key.' + required: true + path: + description: 'File, directory or glob to stash.' + required: true + if-no-files-found: + description: "'error' fails when path matches nothing, 'warn' or 'ignore' do not." + required: false + default: 'warn' + retention-days: + description: > + Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's + S3CacheExpirationInDays lifecycle rule expires the tarball itself. + required: false + default: '5' + compression-level: + description: "Zlib compression level (0-9). On RunsOn, '0' stores the tarball uncompressed." + required: false + default: '6' +runs: + using: 'composite' + steps: + - name: "Save stash to GitHub artifacts" + if: runner.environment != 'self-hosted' + uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + with: + key: ${{ inputs.key }} + path: ${{ inputs.path }} + if-no-files-found: ${{ inputs.if-no-files-found }} + retention-days: ${{ inputs.retention-days }} + compression-level: ${{ inputs.compression-level }} + - name: "Save stash to S3" + id: s3 + if: runner.environment == 'self-hosted' + shell: bash + env: + STASH_KEY: ${{ inputs.key }} + STASH_PATH: ${{ inputs.path }} + DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }} + IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }} + COMPRESSION_LEVEL: ${{ inputs.compression-level }} + run: | + set -euo pipefail + bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}" + export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}" + shopt -s nullglob + # shellcheck disable=SC2206 + candidates=( ${STASH_PATH} ) + existing=() + for p in "${candidates[@]}"; do + if [[ -e "${p}" ]]; then + existing+=("${p}") + fi + done + if [[ ${#existing[@]} -eq 0 ]]; then + if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then + echo "::error::No files found for path '${STASH_PATH}'" + exit 1 + fi + if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then + echo "::warning::No files found for path '${STASH_PATH}', skipping save" + fi + exit 0 + fi + # The tarball is streamed to S3 while being hashed: a multi-GB image never touches the + # runner's disk. Its key is only known at the end, so it lands under a temporary key and + # is copied server-side to its digest key. + prefix="s3://${bucket}/cache/airflow-stash" + upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar" + digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo" + digest_file="${RUNNER_TEMP}/stash-save-$$.sha256" + trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT + mkfifo "${digest_fifo}" + sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" & + digest_pid=$! + compression=() + if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then + compression=(-z) + fi + tar -cP "${compression[@]}" -f - "${existing[@]}" \ + | tee "${digest_fifo}" \ + | aws s3 cp --only-show-errors - "${upload}" + wait "${digest_pid}" + digest="$(cat "${digest_file}")" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "::error::Could not hash the stash '${STASH_KEY}'" + exit 1 + fi + object="${prefix}/sha256/${digest}.tar" + aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}" + # Leaving the upload behind only costs storage until the bucket's lifecycle rule expires it. + aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not remove ${upload}" + mkdir -p "${DIGEST_DIR}" + echo "${digest}" > "${DIGEST_DIR}/sha256" + echo "saved=true" >> "${GITHUB_OUTPUT}" + echo "Saved ${existing[*]} to ${object}" + - name: "Save stash digest to GitHub artifacts" + if: runner.environment == 'self-hosted' && steps.s3.outputs.saved == 'true' + uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + with: + key: ${{ inputs.key }}-s3 + path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index de26e37fbfc..7d60cf6ef41 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -62,7 +62,7 @@ jobs: # layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners. timeout-minutes: 35 name: Check that image builds quickly - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index dc5da4b211e..c850b783deb 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -152,7 +152,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of PROD image building" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -189,7 +189,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -220,7 +220,7 @@ jobs: task-sdk-integration-tests: timeout-minutes: 60 name: "Task SDK integration tests with PROD image" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -254,7 +254,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Regular e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -264,7 +264,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -276,7 +276,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Elasticsearch remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -288,7 +288,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "OpenSearch remote logging e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -300,7 +300,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "XCom object storage backend e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -311,7 +311,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Event driven e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -323,7 +323,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Java SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -335,7 +335,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Go SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -347,7 +347,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "TypeScript SDK e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -359,7 +359,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "OpenLineage e2e test" - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -377,7 +377,7 @@ jobs: compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} uses: ./.github/workflows/openlineage-e2e-compat-tests.yml with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -423,7 +423,7 @@ jobs: airflow-ctl-integration-tests: timeout-minutes: 60 name: "Airflow CTL integration tests with PROD image" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 4212f4bd872..88740c02c46 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -87,7 +87,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} steps: - name: "Cleanup repo" shell: bash @@ -105,7 +105,7 @@ jobs: run-breeze-integration-tests: timeout-minutes: 120 name: Breeze integration tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-breeze-integration-tests == 'true' steps: - name: "Cleanup repo" @@ -136,13 +136,15 @@ jobs: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. UV_PYTHON: "${{ inputs.default-python-version }}" SHARED_DISTRIBUTIONS: ${{ join(fromJSON(inputs.shared-distributions-as-json), ' ') }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 1 persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = ""`. @@ -171,7 +173,7 @@ jobs: tests-scripts: timeout-minutes: 10 name: Scripts tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-scripts-tests == 'true' env: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. @@ -182,6 +184,8 @@ jobs: with: fetch-depth: 1 persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = ""`. @@ -200,7 +204,7 @@ jobs: tests-ui: timeout-minutes: 15 name: React UI tests - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.run-ui-tests == 'true' steps: - name: "Cleanup repo" @@ -210,6 +214,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -237,7 +243,7 @@ jobs: check-translation-completness: timeout-minutes: 15 name: "Check translation completeness" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -254,7 +260,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -327,7 +333,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -381,7 +387,7 @@ jobs: test-airflow-standalone: timeout-minutes: 30 name: "Test Airflow standalone commands" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: AIRFLOW_HOME: ~/airflow FORCE_COLOR: 1 @@ -390,6 +396,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Install uv" # Extract uv version from uv.lock. The format is stable: the line # immediately after `name = "uv"` is `version = ""`. diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 92f1903a1b7..05fa7900c0f 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -20,8 +20,9 @@ # `release-branch-ci.yml`. Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the # scheduled canary. The two workflows are physical copies of each other # (GitHub Actions has no cross-file YAML include), differing only in their -# trigger surface, the hardcoded `runner-type` / `platform` values in -# `build-info`'s outputs, and the architecture-specific print-platform job. +# trigger surface, the hardcoded `platform` value in `build-info`'s outputs +# (and the `PLATFORM` selective checks pick the runner type for), and the +# architecture-specific print-platform job. # Edit both files together when changing pipeline shape. name: Tests (AMD) on: # yamllint disable-line rule:truthy @@ -305,7 +306,7 @@ jobs: latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - # platform / runner-type are hardcoded per workflow file (this is the + # platform is hardcoded per workflow file (this is the # AMD copy). The matching ARM copy lives in ci-arm.yml. platform: "linux/amd64" postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} @@ -347,7 +348,7 @@ jobs: run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} - runner-type: '["ubuntu-22.04"]' + runner-type: ${{ steps.selective-checks.outputs.runner-type }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} @@ -429,6 +430,8 @@ jobs: COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + # Repository variable: "true" moves every run, pushes included, back to public runners. + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" PLATFORM: "linux/amd64" SELECTIVE_TESTS_ON_PUSH: "true" PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }} @@ -566,7 +569,7 @@ jobs: timeout-minutes: 45 name: "MyPy providers checks" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} if: needs.build-info.outputs.run-mypy-providers == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -607,7 +610,7 @@ jobs: timeout-minutes: 20 name: "Migration round-trip check" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.has-migrations == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -729,7 +732,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -758,7 +761,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -787,7 +790,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -817,7 +820,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -846,7 +849,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -877,7 +880,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -908,7 +911,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -938,7 +941,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -1024,7 +1027,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -1054,7 +1057,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -1165,7 +1168,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1181,7 +1184,7 @@ jobs: tests-go-sdk: name: "Go SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1197,6 +1200,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -1235,7 +1240,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1245,6 +1250,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -1269,7 +1276,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }} timeout-minutes: 15 permissions: contents: read @@ -1285,6 +1292,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup Java uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 with: @@ -1299,7 +1308,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 30 permissions: contents: read @@ -1331,7 +1340,7 @@ jobs: build-ts-sdk-docs: name: "TypeScript SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} timeout-minutes: 30 permissions: contents: read @@ -1368,7 +1377,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1389,7 +1398,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -1669,7 +1678,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" @@ -1679,6 +1688,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 180182647a6..1bde91c7aca 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -20,9 +20,10 @@ # `release-branch-ci.yml` — never on PRs. Companion to `ci-amd.yml`, which runs the # same pipeline on AMD for every PR. The two workflows are physical # copies of each other (GitHub Actions has no cross-file YAML include), -# differing only in their trigger surface, the hardcoded `runner-type` / -# `platform` values in `build-info`'s outputs, and the architecture-specific -# print-platform job. Edit both files together when changing pipeline shape. +# differing only in their trigger surface, the hardcoded `platform` value in +# `build-info`'s outputs (and the `PLATFORM` selective checks pick the runner +# type for), and the architecture-specific print-platform job. Edit both files +# together when changing pipeline shape. name: Tests (ARM) on: # yamllint disable-line rule:truthy schedule: @@ -187,7 +188,7 @@ jobs: latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - # platform / runner-type are hardcoded per workflow file (this is the + # platform is hardcoded per workflow file (this is the # ARM copy). The matching AMD copy lives in ci-amd.yml. platform: "linux/arm64" postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} @@ -229,7 +230,7 @@ jobs: run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} - runner-type: '["ubuntu-22.04-arm"]' + runner-type: ${{ steps.selective-checks.outputs.runner-type }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} @@ -311,6 +312,8 @@ jobs: COMMIT_REF: "${{ github.sha }}" VERBOSE: "false" GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + # Repository variable: "true" moves every run, pushes included, back to public runners. + USE_PUBLIC_RUNNERS: "${{ vars.USE_PUBLIC_RUNNERS }}" PLATFORM: "linux/arm64" SELECTIVE_TESTS_ON_PUSH: "true" PUSH_BASE_COMMIT: ${{ needs.check-run-needed.outputs.push-base-commit }} @@ -448,7 +451,7 @@ jobs: timeout-minutes: 45 name: "MyPy providers checks" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} if: needs.build-info.outputs.run-mypy-providers == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -489,7 +492,7 @@ jobs: timeout-minutes: 20 name: "Migration round-trip check" needs: [build-info, build-ci-images] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.has-migrations == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" @@ -611,7 +614,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -640,7 +643,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -669,7 +672,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -699,7 +702,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -728,7 +731,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -759,7 +762,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -790,7 +793,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -820,7 +823,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "" @@ -906,7 +909,7 @@ jobs: if: > needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -936,7 +939,7 @@ jobs: packages: read if: needs.build-info.outputs.run-unit-tests == 'true' with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} test-name: "LowestDeps" force-lowest-dependencies: "true" @@ -1047,7 +1050,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 32, 128) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1063,7 +1066,7 @@ jobs: tests-go-sdk: name: "Go SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1079,6 +1082,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary # keep this in sync with go.mod in go-sdk/ - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -1117,7 +1122,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 15 permissions: contents: read @@ -1127,6 +1132,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: @@ -1151,7 +1158,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 32, 128) }} timeout-minutes: 15 permissions: contents: read @@ -1167,6 +1174,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: Setup Java uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 with: @@ -1181,7 +1190,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} timeout-minutes: 30 permissions: contents: read @@ -1213,7 +1222,7 @@ jobs: build-ts-sdk-docs: name: "TypeScript SDK docs" needs: [build-info] - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 4, 16) }} timeout-minutes: 30 permissions: contents: read @@ -1250,7 +1259,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 4, 16) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1271,7 +1280,7 @@ jobs: contents: read packages: read with: - runners: ${{ needs.build-info.outputs.runner-type }} + runners: ${{ format(needs.build-info.outputs.runner-type, 8, 32) }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -1550,7 +1559,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], 8, 32) }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" @@ -1560,6 +1569,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 315256420fd..23f8e1f724c 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -128,7 +128,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) }} timeout-minutes: 110 name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} @@ -147,24 +147,38 @@ jobs: with: ref: ${{ inputs.checkout-ref }} persist-credentials: false + # A local action is read from the workspace, and `checkout-ref` may be a release ref cut + # before the stash actions existed, so they are loaded from the workflow's own version. The + # runner-preparing scripts too: an older `make_mnt_writeable.sh` empties /mnt, which on + # RunsOn holds the runner's own disks. + - name: "Checkout the workflow version for its actions and runner scripts" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + path: workflow-version + sparse-checkout: | + .github/actions + scripts/ci - name: "Free up disk space" shell: bash run: ./scripts/tools/free_up_disk_space.sh - name: "Make /mnt writeable" - run: ./scripts/ci/make_mnt_writeable.sh + run: ./workflow-version/scripts/ci/make_mnt_writeable.sh - name: "Move docker to /mnt" - run: ./scripts/ci/move_docker_to_mnt.sh + run: ./workflow-version/scripts/ci/move_docker_to_mnt.sh - name: "Install Breeze" uses: ./.github/actions/breeze # A ref is not a commit: a branch documented twice moves between the two publishes, and # an image built before it moved is a cache, not an answer. The commit each stashed image # is saved with tells the two apart, and it is stashed separately so that deciding costs # a few bytes rather than the multi-gigabyte image the decision may make unnecessary. + # `only-current-branch` must stay true on the image restores below: runs on RunsOn (canaries + # and release-branch pushes) save these image stashes as S3 digests only, so a fallback to the + # base branch would find nothing on public runners anyway. - name: > Restore the commit the CI image stashed for ref ${{ inputs.image-stash-ref }} was built from - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -197,8 +211,7 @@ jobs: - name: > Restore CI docker image built for ref ${{ inputs.image-stash-ref }} ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -244,8 +257,7 @@ jobs: --cache-file /tmp/ci-cache-mount-save-v4-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz if: steps.restore-cache-mount.outputs.stash-hit == 'true' - name: "Restore CI docker image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1 + uses: ./workflow-version/.github/actions/stash-restore with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}" path: "/mnt/" @@ -336,7 +348,7 @@ jobs: shell: bash if: inputs.upload-image-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" @@ -350,7 +362,7 @@ jobs: # what a later publish of that same ref reads - hence a retention that outlives the shared # one, the same ref being rebuilt three days apart at the very least. - name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" @@ -364,7 +376,7 @@ jobs: # Saved last and with the image's retention, so that finding this commit is enough to # know the image it describes is there to be restored. - name: "Stash the commit the CI image for ref ${{ inputs.image-stash-ref }} was built from" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./workflow-version/.github/actions/stash-save with: key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ -${{ inputs.image-stash-ref }}" diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index affe6ebdb8d..1a1c06c6804 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -119,7 +119,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -175,7 +175,7 @@ jobs: build-docs: timeout-minutes: 150 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} if: inputs.docs-build == 'true' env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -334,7 +334,7 @@ jobs: id-token: write contents: read needs: build-docs - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -466,7 +466,7 @@ jobs: test-python-api-client: timeout-minutes: 60 name: "Test Python API client" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} if: inputs.run-api-codegen == 'true' env: BACKEND: "postgres" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 043aa8f0ebf..b664e49cf06 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -103,6 +103,8 @@ jobs: with: # Needed to perform push action persist-credentials: false + - name: "RunsOn cost summary" + uses: ./.github/actions/runs-on-cost-summary - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} @@ -124,7 +126,7 @@ jobs: # Constraint commits are now handled by the dedicated update-constraints-on-push workflow # triggered when uv.lock changes are pushed to main/test branches. dependency-upgrade-summary: - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} needs: [update-constraints] if: inputs.upgrade-to-newer-dependencies == 'true' && inputs.platform == 'linux/amd64' name: "Deps ${{ matrix.python-version }}:${{ matrix.constraints-mode }}" diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 9846e44ce71..58312ea63c4 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -85,7 +85,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints for ${{ matrix.python-version }} on ${{ inputs.platform }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -114,6 +114,8 @@ jobs: platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} + # Runs the checkout-ref's own make_mnt_writeable.sh, which on old refs empties /mnt and + # would kill a RunsOn runner: every caller passing checkout-ref pins public runners. make-mnt-writeable-and-cleanup: true - name: "Install prek" uses: ./.github/actions/install-prek diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 7e779afcc2a..2cbe0603a43 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -52,7 +52,7 @@ jobs: name: >- Unit tests Helm: ${{ matrix.helm-test-package }} (K8S ${{ matrix.kubernetes-version }}) - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} strategy: fail-fast: false max-parallel: 20 @@ -98,7 +98,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 202d70cd9ef..efc8d578b5c 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -75,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 32, 128) }} strategy: fail-fast: false max-parallel: 20 @@ -126,7 +126,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true' name: "Integration: providers ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -176,7 +176,7 @@ jobs: timeout-minutes: 30 if: inputs.run-system-tests == 'true' name: "System Tests" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} env: BACKEND: "postgres" BACKEND_VERSION: ${{ inputs.default-postgres-version }} diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index d295b9d713d..39b0277e110 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -62,7 +62,7 @@ jobs: tests-kubernetes: timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false max-parallel: 20 @@ -149,7 +149,7 @@ jobs: tests-kubernetes-lang-sdk: timeout-minutes: 60 name: "K8S Lang-SDK:${{ inputs.lang-sdk-kubernetes-combo }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} INCLUDE_SUCCESS_OUTPUTS: ${{ inputs.include-success-outputs }} diff --git a/.github/workflows/kustomize-overlays-tests.yml b/.github/workflows/kustomize-overlays-tests.yml index 1cca2b85000..5fef9c1ac8a 100644 --- a/.github/workflows/kustomize-overlays-tests.yml +++ b/.github/workflows/kustomize-overlays-tests.yml @@ -53,7 +53,7 @@ jobs: smoke-test-overlay: timeout-minutes: 40 name: "Overlay smoke test: ${{ matrix.overlay }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 4, 16) }} strategy: fail-fast: false max-parallel: 4 diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index f136cd1ce61..31eaabace18 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -119,7 +119,7 @@ jobs: build-prod-packages: name: "Build Airflow and provider distributions" timeout-minutes: 10 - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -199,7 +199,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) }} timeout-minutes: 80 name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} needs: - build-prod-packages env: @@ -289,7 +289,7 @@ jobs: breeze prod-image save --platform "${PLATFORM}" --image-file-dir "/mnt" if: inputs.upload-image-artifact == 'true' - name: "Stash PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2 + uses: ./.github/actions/stash-save with: key: prod-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} path: "/mnt/prod-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 4d04deb09da..3dc4693e0df 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -79,7 +79,7 @@ on: # yamllint disable-line rule:truthy jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} permissions: contents: read packages: write @@ -149,7 +149,7 @@ jobs: push-prod-image-cache: name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} permissions: contents: read packages: write diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 6ac4d60d1ba..ff56fbb9df5 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -94,7 +94,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -119,7 +119,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -144,7 +144,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -169,7 +169,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -194,7 +194,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -220,7 +220,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -246,7 +246,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" @@ -272,7 +272,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 8, 32) }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" @@ -298,7 +298,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" @@ -323,7 +323,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 4, 16) }} platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" @@ -348,7 +348,7 @@ jobs: contents: read packages: read with: - runners: ${{ inputs.runners }} + runners: ${{ format(inputs.runners, 32, 128) }} platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index e7b3575b3f1..406ebde52fa 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -72,7 +72,7 @@ jobs: prepare-install-verify-provider-distributions: timeout-minutes: 80 name: "Providers ${{ matrix.package-format }} tests" - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false matrix: @@ -178,7 +178,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.test-types.description }} - runs-on: ${{ fromJSON(inputs.runners) }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml index 8b8ad21e4bf..1ee5e1e6a67 100644 --- a/.github/workflows/ui-e2e-tests.yml +++ b/.github/workflows/ui-e2e-tests.yml @@ -88,7 +88,7 @@ jobs: test-ui-e2e-tests: timeout-minutes: 90 name: ${{ inputs.workflow-name || 'UI E2E Tests' }} - runs-on: ${{ fromJSON(inputs.runners || '["ubuntu-24.04"]') }} + runs-on: ${{ format(fromJSON(inputs.runners || '["ubuntu-24.04"]')[0], 8, 32) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version || '3.11' }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/README.md b/README.md index 8e7f8f70b41..e1a994222e9 100644 --- a/README.md +++ b/README.md @@ -603,3 +603,5 @@ The CI infrastructure for Apache Airflow has been sponsored by: astronomer.io AWS OpenSource + +The self-hosted CI runners on AWS are managed by [RunsOn](https://runs-on.com), which is free for open-source projects. diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 2356d61c3ea..d15ff81dd5e 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -33,7 +33,7 @@ - [Skipping prek hooks (Static checks)](#skipping-prek-hooks-static-checks) - [Suspended providers](#suspended-providers) - [Selective check outputs](#selective-check-outputs) - - [Committer vs. Non-committer PRs](#committer-vs-non-committer-prs) + - [Self-hosted (RunsOn) vs. public runners](#self-hosted-runson-vs-public-runners) - [Changing behaviours of the CI runs by setting labels](#changing-behaviours-of-the-ci-runs-by-setting-labels) @@ -689,6 +689,7 @@ GitHub Actions to pass the list of parameters to a command to execute | run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | | amd-runners | List of labels assigned for runners for that build for public AMD runners | \["ubuntu-22.04"\] | | | arm-runners | List of labels assigned for runners for that build for public ARM runners | \["ubuntu-22.04-arm"\] | | +| runner-type | Runner labels for the CI jobs on this platform: public, or a RunsOn template jobs fill with cpu/ram | \["ubuntu-22.04"\] | | | selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | | skip-prek-hooks | Which prek hooks should be skipped during the static-checks run | flynt,identity | | | skip-providers-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | @@ -698,27 +699,82 @@ GitHub Actions to pass the list of parameters to a command to execute | upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | -## Committer vs. Non-committer PRs +## Self-hosted (RunsOn) vs. public runners -There is a difference in how the CI jobs are run for committer and non-committer PRs from forks. -The main reason is security; we do not want to run untrusted code on our infrastructure for self-hosted runners. +The `Tests (AMD)` / `Tests (ARM)` jobs run either on GitHub-hosted ("public") runners or on self-hosted +[RunsOn](https://runs-on.com) EC2 runners in the Airflow AWS account. Selective checks route runs of +code from `apache/airflow` itself there; this is a routing preference, not what keeps fork code off +RunsOn (see below). The `runner-type` output is decided as follows, first match wins: -Currently there is no difference because we are not using `self-hosted` runners (until we implement `Action -Runner Controller` but most of the jobs, committer builds will use "Self-hosted" runners by default, -while non-committer builds will use "Public" runners. For committers, this can be overridden by setting the -`use public runners` label in the PR. +1. Runs in any repository other than `apache/airflow` (forks running their own CI) use public runners. +2. Any run uses public runners while the repository variable `USE_PUBLIC_RUNNERS` is `true`: the + switch for moving `main` off RunsOn without a PR, e.g. when the stack is unavailable. +3. PRs with the `use public runners` label use public runners. +4. Any `push`, `schedule` or `workflow_dispatch` in `apache/airflow` (canary runs and the release + branch pushes, all triggered by people with write access) uses RunsOn. +5. Other events than `pull_request` use public runners. +6. PRs with the `use self-hosted runners` label use RunsOn, if a committer's (`COMMITTERS` in + `global_constants.py`) latest review of the PR's head commit is an approval and no committer's + request for changes stands. The label outlives the commit it was added for, so on its own it would + vouch for every later push: a new push sends the PR back to public runners until the head commit is + approved again, and so does a dismissed approval if no other committer's approval of the head commit + remains. A committer's request for changes keeps the PR on public runners across pushes, as it blocks + a merge on GitHub, until that committer reviews again or the review is dismissed. The label must + also still be on the PR: a re-run reuses the labels of its original event, so they are checked + against the PR's current ones. If the PR or its reviews cannot be read, the label is ignored as well. +7. PRs with the `non committer build` label use public runners. +8. PRs opened by a bot (Dependabot and the like), or from a `dependabot/` branch, use public runners: + they carry code nobody has reviewed yet. +9. PRs from a branch of `apache/airflow` use RunsOn; PRs from forks use public runners. + +Labels are read from the event that started the run, and the CI workflows do not run on `labeled`. After +adding or removing a label, push a commit or close and reopen the PR; re-running an earlier run reuses +its original labels, and "Re-run failed jobs" also reuses that run's runner decision. A push voids the +approval `use self-hosted runners` needs, so for that label approve the head commit and then close and +reopen the PR. + +The RunsOn value is a label template such as +`["runs-on=/family=.../image=ubuntu22-full-x64/volume=150gb/cpu={0}/ram={1}"]`. Every job +picks its size class with `format(fromJSON(inputs.runners)[0], , )` (4/16, 8/32 or 32/128), +or, for a single-job reusable workflow, its caller passes `format(, , )`; +on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are +gated on `runner.environment == 'self-hosted'`: + +* The EC2 cost summary of each job. +* The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to + the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact. + Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by + the same run, branch and repository (or, for a PR, its base branch). The smaller caches (mount cache, prek, docs inventory) use + the ASF stash action on both kinds of runners, so PRs on public runners keep the ones canary runs save; + the price is that every RunsOn canary uploads the mount cache from AWS to GitHub. + +The decision above is not a security boundary. Under the `pull_request` event the code that +evaluates it is the PR's own: a fork PR runs its own workflow files and breeze sources, and can name +RunsOn labels in `runs-on:` directly. RunsOn launches a runner for any job of a repository its GitHub +App is installed on, so nothing in this repository can prevent that. Keeping untrusted code away from +anything that matters has to be enforced outside the repository, by: + +* The repository's Actions setting "Approval for running fork pull request workflows from + contributors", which must be "Require approval for all external contributors". That still exempts + members of the `apache` GitHub organization (ASF committers of any project who linked their GitHub + account) and repository collaborators, but every exempt person is identifiable. +* The RunsOn stack, which must be dedicated to this repository and give the runners an instance role + that reaches nothing beyond the stack's own cache bucket (whose contents are never trusted, see the + stash actions), no EBS permissions, no SSH, network isolation between runners and a maximum run + time. RunsOn has no cap on instance sizes or counts and its budget only alerts, so the AWS account's + EC2 vCPU quotas are the cost ceiling. Pushes in a contributor fork are never committer builds, whoever pushes, because forks have no self-hosted runners. ## Changing behaviours of the CI runs by setting labels -Also, currently for most of the jobs, committer builds by default use "Self-hosted" runners, while -non-committer builds use "Public" runners. For committers, this can be overridden by setting the -`use public runners` label in the PR. In the future when we might also switch committers to public runners. -Committers will be able to use `use self-hosted runners` label in the PR to force using self-hosted runners. -The `use public runners` label will still be available for committers and they will be able to set it for -builds that also have `canary` label set to also switch the `canary` builds to public runners. +PRs from `apache/airflow` branches run on self-hosted RunsOn runners and PRs from forks run on public +runners (see above). Set `use public runners` to move a PR to public runners; a committer can set +`use self-hosted runners` and approve the PR's head commit to move a fork PR to RunsOn, provided no +committer's request for changes stands, then close and reopen the PR (a push would need a fresh +approval; a committer should do the reopen, since an external author's reopen still needs a maintainer +to approve the workflow run). If you are testing CI workflow changes and want to test it for more complete matrix combinations generated by the jobs - you can set `all versions` label in the PR. This will run the PRs with the same combinations @@ -726,7 +782,7 @@ of versions as the `canary` main build. Using `all versions` is automatically se change in `pyproject.toml`. If you are running an `apache` PR, you can also set `canary` label for such PR and in this case, all the -`canary` properties of build will be used: `self-hosted` runners, `full tests needed` mode, `all versions` +`canary` properties of build will be used: `full tests needed` mode, `all versions` as well as all canary-specific jobs will run there. You can modify this behaviour of the `canary` run by applying `use public runners`, and `default versions only` labels to the PR as well which will still run a `canary` equivalent build but with public runners an default Python/K8S versions only - respectively. diff --git a/dev/breeze/doc/ci/06_debugging.md b/dev/breeze/doc/ci/06_debugging.md index c87c2b3f5b0..dfe9b968931 100644 --- a/dev/breeze/doc/ci/06_debugging.md +++ b/dev/breeze/doc/ci/06_debugging.md @@ -44,12 +44,16 @@ in some cases, you need to run the PR as coming from the "apache" repository rather than from your fork. You can also apply the label later and rebase the PR or close/reopen -the PR to apply the label to the PR. +the PR to apply the label to the PR. For `use self-hosted runners`, a +committer approves the head commit (with no committer's request for +changes standing) and closes/reopens the PR: a rebase needs a fresh +approval. | Action to perform | Label to set | PR from "apache" repo | |------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------|:---------------------:| | Run the build with all combinations of all
python, backends, kubernetes etc on PR,
and run all types of tests for all test
groups. | full tests needed | | | Force to use public runners for the build | use public runners | | +| Run a PR from a fork on the self-hosted RunsOn
runners (needs a committer's approval of its head commit
and no committer's request for changes) | use self-hosted runners | | | Debug resources used during the build for
parallel jobs | debug ci resources | | | Force running PR on latest versions of
python, backends, kubernetes etc. when you
want to save resources and test only latest
versions | latest versions only | | | Force running PR on minimal (default)
versions of python, backends, kubernetes etc.
in order to save resources and run tests only
for minimum versions | default versions only | | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index d7fec43e8bd..15a98856e5e 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -38,6 +38,14 @@ from airflow_breeze.utils.path_utils import ( PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]' PUBLIC_ARM_RUNNERS = '["ubuntu-22.04-arm"]' +# RunsOn (runs-on.com) EC2 runner label templates. Jobs fill in their size class with +# `format(