airflow/.github/workflows/integration-system-tests.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Integration and system tests
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runners:
        description: "The array of labels (in json form) determining public runners."
        required: true
        type: string
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      testable-core-integrations:
        description: "The list of testable core integrations as JSON array."
        required: true
        type: string
      testable-providers-integrations:
        description: "The list of testable providers integrations as JSON array."
        required: true
        type: string
      run-system-tests:
        description: "Run system tests (true/false)"
        required: true
        type: string
      default-postgres-version:
        description: "Default version of Postgres to use"
        required: true
        type: string
      default-mysql-version:
        description: "Default version of MySQL to use"
        required: true
        type: string
      skip-providers-tests:
        description: "Skip provider tests (true/false)"
        required: true
        type: string
      run-coverage:
        description: "Run coverage (true/false)"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      debug-resources:
        description: "Debug resources (true/false)"
        required: true
        type: string
      use-uv:
        description: "Whether to use uv"
        required: true
        type: string
permissions:
  contents: read
jobs:
  tests-core-integration:
    timeout-minutes: 30
    if: inputs.testable-core-integrations != '[]'
    name: "Integration core ${{ matrix.integration }}"
    runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
    strategy:
      fail-fast: false
      # 20 keeps runs on GitHub-hosted runners within the ASF limit of 20 concurrent jobs. The RunsOn
      # runners are the project's own capacity, so a run there starts the whole matrix at once.
      max-parallel: ${{ contains(inputs.runners, 'runs-on=') && 256 || 20 }}
      matrix:
        integration: ${{ fromJSON(inputs.testable-core-integrations) }}
    env:
      BACKEND: "postgres"
      BACKEND_VERSION: ${{ inputs.default-postgres-version }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      JOB_ID: "integration-core-${{ matrix.integration }}"
      SKIP_PROVIDERS_TESTS: "${{ inputs.skip-providers-tests }}"
      ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: "${{ inputs.default-python-version }}"
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
      - name: "Integration: core ${{ matrix.integration }}"
        env:
          INTEGRATION: "${{ matrix.integration }}"
        # yamllint disable rule:line-length
        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${INTEGRATION}"
      - name: "Post Tests success"
        uses: ./.github/actions/post_tests_success
        with:
          codecov-token: ${{ secrets.CODECOV_TOKEN }}
          python-version: "${{ inputs.default-python-version }}"
      - name: "Post Tests failure"
        uses: ./.github/actions/post_tests_failure
        if: failure()

  tests-providers-integration:
    timeout-minutes: 30
    if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true'
    name: "Integration: providers ${{ matrix.integration }}"
    runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }}
    strategy:
      fail-fast: false
      # 20 keeps runs on GitHub-hosted runners within the ASF limit of 20 concurrent jobs. The RunsOn
      # runners are the project's own capacity, so a run there starts the whole matrix at once.
      max-parallel: ${{ contains(inputs.runners, 'runs-on=') && 256 || 20 }}
      matrix:
        integration: ${{ fromJSON(inputs.testable-providers-integrations) }}
    env:
      BACKEND: "postgres"
      BACKEND_VERSION: ${{ inputs.default-postgres-version }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      JOB_ID: "integration-providers-${{ matrix.integration }}"
      SKIP_PROVIDERS_TESTS: "${{ inputs.skip-providers-tests }}"
      ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: "${{ inputs.default-python-version }}"
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
      - name: "Integration: providers ${{ matrix.integration }}"
        env:
          INTEGRATION: "${{ matrix.integration }}"
        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${INTEGRATION}"
      - name: "Post Tests success"
        uses: ./.github/actions/post_tests_success
        with:
          codecov-token: ${{ secrets.CODECOV_TOKEN }}
          python-version: "${{ inputs.default-python-version }}"
      - name: "Post Tests failure"
        uses: ./.github/actions/post_tests_failure
        if: failure()

  tests-system:
    timeout-minutes: 30
    if: inputs.run-system-tests == 'true'
    name: "System Tests"
    runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }}
    env:
      BACKEND: "postgres"
      BACKEND_VERSION: ${{ inputs.default-postgres-version }}
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      JOB_ID: "system"
      SKIP_PROVIDERS_TESTS: "${{ inputs.skip-providers-tests }}"
      ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
        uses: ./.github/actions/prepare_breeze_and_image
        with:
          platform: ${{ inputs.platform }}
          python: "${{ inputs.default-python-version }}"
          use-uv: ${{ inputs.use-uv }}
          make-mnt-writeable-and-cleanup: true
      - name: "System Tests"
        run: >
          ./scripts/ci/testing/run_system_tests.sh airflow-core/tests/system/example_empty.py
      - name: "Post Tests success"
        uses: ./.github/actions/post_tests_success
        with:
          codecov-token: ${{ secrets.CODECOV_TOKEN }}
          python-version: "${{ inputs.default-python-version }}"
      - name: "Post Tests failure"
        uses: ./.github/actions/post_tests_failure
        if: failure()