mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Jason(Zhe-You) Liu
406474723a
The YAML Dag importer was covered by unit tests only. Nothing ran a YAML Dag through a real Dag processor, and the `run:` plus queue-routing path had no coverage at all. Add a `yaml_dag` e2e mode whose Dag exercises every task body the importer supports: provider operators and a sensor driving S3 on LocalStack, a `run:` task routed by its queue to the Go coordinator, and a bodiless task that becomes an EmptyOperator. The Go example bundle gains the handler behind that one code task.
229 lines
10 KiB
YAML
229 lines
10 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
---
|
|
|
|
name: Airflow E2E Tests
|
|
|
|
permissions:
|
|
contents: read
|
|
on: # yamllint disable-line rule:truthy
|
|
workflow_dispatch:
|
|
inputs:
|
|
workflow-name:
|
|
description: "Name of the test"
|
|
type: string
|
|
required: true
|
|
runners:
|
|
description: "The array of labels (in json form) determining runners."
|
|
type: string
|
|
default: '["ubuntu-24.04"]'
|
|
platform:
|
|
description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
|
|
type: string
|
|
default: 'linux/amd64'
|
|
default-python-version:
|
|
description: "Which version of python should be used by default"
|
|
type: string
|
|
default: '3.11'
|
|
use-uv:
|
|
description: "Whether to use uv to build the image (true/false)"
|
|
type: string
|
|
default: 'true'
|
|
docker-image-tag:
|
|
description: "Tag of the Docker image to test"
|
|
type: string
|
|
required: true
|
|
e2e_test_mode:
|
|
description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, ts_sdk, or yaml_dag" # yamllint disable-line rule:line-length
|
|
type: string
|
|
default: "basic"
|
|
java-sdk-version:
|
|
description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line rule:line-length
|
|
type: string
|
|
default: "17"
|
|
|
|
workflow_call:
|
|
inputs:
|
|
workflow-name:
|
|
description: "Name of the test"
|
|
type: string
|
|
required: true
|
|
runners:
|
|
description: "The array of labels (in json form) determining runners."
|
|
required: true
|
|
type: string
|
|
platform:
|
|
description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
|
|
required: true
|
|
type: string
|
|
default-python-version:
|
|
description: "Which version of python should be used by default"
|
|
required: true
|
|
type: string
|
|
use-uv:
|
|
description: "Whether to use uv to build the image (true/false)"
|
|
required: true
|
|
type: string
|
|
docker-image-tag:
|
|
description: "Tag of the Docker image to test"
|
|
type: string
|
|
default: ""
|
|
e2e_test_mode:
|
|
description: "Test mode - basic, remote_log, remote_log_elasticsearch, remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, go_sdk, ts_sdk, or yaml_dag" # yamllint disable-line rule:line-length
|
|
type: string
|
|
default: "basic"
|
|
java-sdk-version:
|
|
description: "JDK version used to build the Java SDK bundles natively (tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line rule:line-length
|
|
type: string
|
|
default: "17"
|
|
|
|
jobs:
|
|
test-e2e-integration-tests:
|
|
timeout-minutes: 60
|
|
name: ${{ inputs.workflow-name }}
|
|
runs-on: ${{ fromJSON(inputs.runners) }}
|
|
env:
|
|
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
|
|
GITHUB_REPOSITORY: ${{ github.repository }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_USERNAME: ${{ github.actor }}
|
|
VERBOSE: "true"
|
|
steps:
|
|
- name: "Cleanup repo"
|
|
shell: bash
|
|
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
fetch-depth: 2
|
|
persist-credentials: false
|
|
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
|
|
uses: ./.github/actions/prepare_breeze_and_image
|
|
with:
|
|
platform: ${{ inputs.platform }}
|
|
image-type: "prod"
|
|
python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
|
|
use-uv: ${{ inputs.use-uv }}
|
|
make-mnt-writeable-and-cleanup: true
|
|
id: breeze
|
|
# The java_sdk mode runs the Gradle builds, the go_sdk mode runs the Go
|
|
# build and the ts_sdk mode runs the pnpm build; every other e2e mode
|
|
# skips the toolchain steps below.
|
|
# LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the lang-SDK k8s job uses)
|
|
# makes the e2e conftest build with the host toolchain provisioned here
|
|
# instead of an ephemeral toolchain container, skipping the image pull; local
|
|
# runs keep the containerized builds. The cache keys carry a ``-vN-`` salt
|
|
# (bump to force-invalidate a poisoned cache) and runner.arch keys each cache
|
|
# per architecture, as this job runs on amd64 and arm64. Without its cache
|
|
# each java_sdk run re-downloads the example bundles' full dependency tree
|
|
# (Spark alone is ~1GB) into a fresh ~/.gradle, whose ~/.gradle/jdks holds
|
|
# arch-specific auto-provisioned JDKs (the example pins a Java 11 toolchain).
|
|
- name: "Setup Java for the Java SDK build"
|
|
if: inputs.e2e_test_mode == 'java_sdk'
|
|
uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
|
|
with:
|
|
distribution: 'temurin'
|
|
java-version: ${{ inputs.java-sdk-version }}
|
|
- name: "Restore Java SDK Gradle dependency cache"
|
|
if: inputs.e2e_test_mode == 'java_sdk'
|
|
id: gradle-cache
|
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
~/.gradle/jdks
|
|
!~/.gradle/caches/*.lock
|
|
!~/.gradle/caches/journal-1
|
|
key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties', 'java-sdk/gradle/wrapper/gradle-wrapper.properties', 'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line rule:line-length
|
|
restore-keys: |
|
|
e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-
|
|
# go-version-file sources the Go toolchain from go-sdk/go.mod, so there is no
|
|
# separate version pin to keep in sync; cache: false because the explicit
|
|
# restore/save pair below persists the module + build caches even on red runs.
|
|
- name: "Setup Go for the Go SDK build"
|
|
if: inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag'
|
|
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
|
with:
|
|
go-version-file: go-sdk/go.mod
|
|
cache: false
|
|
- name: "Restore Go SDK module + build cache"
|
|
if: inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag'
|
|
id: go-cache
|
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go-sdk/go.mod', 'go-sdk/go.sum') }} # yamllint disable-line rule:line-length
|
|
restore-keys: |
|
|
e2e-go-sdk-go-v1-${{ runner.os }}-${{ runner.arch }}-
|
|
- name: "Setup pnpm for the TypeScript SDK build"
|
|
if: inputs.e2e_test_mode == 'ts_sdk'
|
|
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
|
with:
|
|
package_json_file: ts-sdk/package.json
|
|
run_install: false
|
|
- name: "Setup Node for the TypeScript SDK build"
|
|
if: inputs.e2e_test_mode == 'ts_sdk'
|
|
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
|
with:
|
|
node-version: 22
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'ts-sdk/pnpm-lock.yaml'
|
|
- name: "Test e2e integration tests"
|
|
run: breeze testing airflow-e2e-tests
|
|
env:
|
|
DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
|
|
E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
|
|
LANG_SDK_NATIVE_TOOLCHAIN: "${{ (inputs.e2e_test_mode == 'java_sdk' || inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'ts_sdk' || inputs.e2e_test_mode == 'yaml_dag') && 'true' || 'false' }}" # yamllint disable-line rule:line-length
|
|
- name: "Save Java SDK Gradle dependency cache"
|
|
# Saved even when the e2e tests fail: the Gradle warm-up is independent of
|
|
# test outcome, and actions/cache's post step would drop it on every red run.
|
|
if: always() && inputs.e2e_test_mode == 'java_sdk' && steps.gradle-cache.outputs.cache-hit != 'true'
|
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: |
|
|
~/.gradle/caches
|
|
~/.gradle/wrapper
|
|
~/.gradle/jdks
|
|
!~/.gradle/caches/*.lock
|
|
!~/.gradle/caches/journal-1
|
|
key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
|
|
- name: "Save Go SDK module + build cache"
|
|
# Saved even when the e2e tests fail, for the same reason as the Gradle cache above.
|
|
if: always() && (inputs.e2e_test_mode == 'go_sdk' || inputs.e2e_test_mode == 'yaml_dag') && steps.go-cache.outputs.cache-hit != 'true' # yamllint disable-line rule:line-length
|
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
|
with:
|
|
path: |
|
|
~/.cache/go-build
|
|
~/go/pkg/mod
|
|
key: ${{ steps.go-cache.outputs.cache-primary-key }}
|
|
- name: Zip logs
|
|
run: |
|
|
cd ./airflow-e2e-tests && zip -r logs.zip logs
|
|
if: always()
|
|
- name: "Upload logs"
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: "e2e-test-logs-${{ inputs.e2e_test_mode }}"
|
|
path: './airflow-e2e-tests/logs.zip'
|
|
retention-days: '7'
|
|
if-no-files-found: 'error'
|
|
if: always()
|