mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Vincent
acb2208f15
The ECS executor calls RunTask for one task at a time, inside the scheduler loop, and waits for each answer. A call takes about 0.7 s, so a scheduler starts at most about 1.4 tasks per second however many slots are free, and its loop stalls while it works through a backlog: scheduling, heartbeats and executor events all wait, for up to 13 minutes in a test with 10,000 tasks. A throttled RunTask call no longer uses one of the task's attempts, since the throttling says nothing about the task; parallel calls make it more likely.
1645 lines
67 KiB
YAML
1645 lines
67 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
---
|
|
package-name: apache-airflow-providers-amazon
|
|
name: Amazon
|
|
description: |
|
|
Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
|
|
|
|
state: ready
|
|
lifecycle: production
|
|
source-date-epoch: 1792489528
|
|
# Note that those versions are maintained by release manager - do not update them manually
|
|
# with the exception of case where other provider in sources has >= new provider version.
|
|
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
# to be done in the same PR
|
|
versions:
|
|
- 9.39.0
|
|
- 9.38.0
|
|
- 9.37.0
|
|
- 9.36.0
|
|
- 9.35.1
|
|
- 9.35.0
|
|
- 9.34.0
|
|
- 9.33.0
|
|
- 9.32.0
|
|
- 9.31.0
|
|
- 9.30.0
|
|
- 9.29.0
|
|
- 9.28.0
|
|
- 9.27.0
|
|
- 9.26.0
|
|
- 9.25.0
|
|
- 9.24.0
|
|
- 9.23.0
|
|
- 9.22.0
|
|
- 9.21.0
|
|
- 9.20.0
|
|
- 9.19.0
|
|
- 9.18.1
|
|
- 9.18.0
|
|
- 9.17.0
|
|
- 9.16.0
|
|
- 9.15.0
|
|
- 9.14.0
|
|
- 9.13.0
|
|
- 9.12.0
|
|
- 9.11.0
|
|
- 9.10.0
|
|
- 9.9.0
|
|
- 9.8.0
|
|
- 9.7.0
|
|
- 9.6.1
|
|
- 9.6.0
|
|
- 9.5.0
|
|
- 9.4.0
|
|
- 9.2.0
|
|
- 9.1.0
|
|
- 9.0.0
|
|
- 8.29.0
|
|
- 8.28.0
|
|
- 8.27.0
|
|
- 8.26.0
|
|
- 8.25.0
|
|
- 8.24.0
|
|
- 8.23.0
|
|
- 8.22.0
|
|
- 8.21.0
|
|
- 8.20.0
|
|
- 8.19.0
|
|
- 8.18.0
|
|
- 8.17.0
|
|
- 8.16.0
|
|
- 8.15.0
|
|
- 8.14.0
|
|
- 8.13.0
|
|
- 8.12.0
|
|
- 8.11.0
|
|
- 8.10.0
|
|
- 8.9.0
|
|
- 8.8.0
|
|
- 8.7.1
|
|
- 8.7.0
|
|
- 8.6.0
|
|
- 8.5.1
|
|
- 8.5.0
|
|
- 8.4.0
|
|
- 8.3.1
|
|
- 8.3.0
|
|
- 8.2.0
|
|
- 8.1.0
|
|
- 8.0.0
|
|
- 7.4.1
|
|
- 7.4.0
|
|
- 7.3.0
|
|
- 7.2.1
|
|
- 7.2.0
|
|
- 7.1.0
|
|
- 7.0.0
|
|
- 6.2.0
|
|
- 6.1.0
|
|
- 6.0.0
|
|
- 5.1.0
|
|
- 5.0.0
|
|
- 4.1.0
|
|
- 4.0.0
|
|
- 3.4.0
|
|
- 3.3.0
|
|
- 3.2.0
|
|
- 3.1.1
|
|
- 3.0.0
|
|
- 2.6.0
|
|
- 2.5.0
|
|
- 2.4.0
|
|
- 2.3.0
|
|
- 2.2.0
|
|
- 2.1.0
|
|
- 2.0.0
|
|
- 1.4.0
|
|
- 1.3.0
|
|
- 1.2.0
|
|
- 1.1.0
|
|
- 1.0.0
|
|
|
|
integrations:
|
|
- integration-name: Amazon Athena
|
|
external-doc-url: https://aws.amazon.com/athena/
|
|
logo: /docs/integration-logos/Amazon-Athena_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/athena/athena_boto.rst
|
|
- /docs/apache-airflow-providers-amazon/operators/athena/athena_sql.rst
|
|
- /docs/apache-airflow-providers-amazon/operators/athena/athena_spark.rst
|
|
tags: [aws]
|
|
# DuckDB itself is not an AWS service: the generic hook, operator and ``duckdb`` connection type
|
|
# live in the ``duckdb`` provider. What lives here is only the AWS-specific part, brokering AWS
|
|
# credentials and endpoints into a DuckDB session configured by its own connection type
|
|
- integration-name: DuckDB on AWS
|
|
external-doc-url: https://duckdb.org/docs/stable/extensions/httpfs/s3api
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/duckdb.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Bedrock
|
|
external-doc-url: https://aws.amazon.com/bedrock/
|
|
logo: /docs/integration-logos/Amazon-Bedrock_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/bedrock.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Chime
|
|
external-doc-url: https://aws.amazon.com/chime/
|
|
logo: /docs/integration-logos/Amazon-Chime-light-bg.png
|
|
tags: [aws]
|
|
- integration-name: Amazon CloudFormation
|
|
external-doc-url: https://aws.amazon.com/cloudformation/
|
|
logo: /docs/integration-logos/AWS-CloudFormation_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/cloudformation.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon CloudWatch Logs
|
|
external-doc-url: https://aws.amazon.com/cloudwatch/
|
|
logo: /docs/integration-logos/Amazon-CloudWatch_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Comprehend
|
|
external-doc-url: https://aws.amazon.com/comprehend/
|
|
logo: /docs/integration-logos/Amazon-Comprehend_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/comprehend.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon DataSync
|
|
external-doc-url: https://aws.amazon.com/datasync/
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/datasync.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon DynamoDB
|
|
external-doc-url: https://aws.amazon.com/dynamodb/
|
|
logo: /docs/integration-logos/Amazon-DynamoDB_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/dynamodb.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon EC2
|
|
external-doc-url: https://aws.amazon.com/ec2/
|
|
logo: /docs/integration-logos/Amazon-EC2_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/ec2.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Elastic Container Registry (ECR)
|
|
external-doc-url: https://aws.amazon.com/ecr/
|
|
logo: /docs/integration-logos/Amazon-Elastic-Container-Registry_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/ecr.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon ECS
|
|
external-doc-url: https://aws.amazon.com/ecs/
|
|
logo: /docs/integration-logos/Amazon-Elastic-Container-Service_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/ecs.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Elastic Kubernetes Service (EKS)
|
|
external-doc-url: https://aws.amazon.com/eks/
|
|
logo: /docs/integration-logos/Amazon-Elastic-Kubernetes-Service_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/eks.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon ElastiCache
|
|
external-doc-url: https://aws.amazon.com/elasticache/redis//
|
|
logo: /docs/integration-logos/Amazon-ElastiCache_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon EMR
|
|
external-doc-url: https://aws.amazon.com/emr/
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/emr/emr.rst
|
|
logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon EMR on EKS
|
|
external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/emr-eks.html
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst
|
|
logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon EMR Serverless
|
|
external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/emr-serverless.html
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst
|
|
logo: /docs/integration-logos/Amazon-EMR_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon EventBridge
|
|
external-doc-url: https://docs.aws.amazon.com/eventbridge/latest/APIReference/Welcome.html
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/eventbridge.rst
|
|
logo: /docs/integration-logos/Amazon-EventBridge_64.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Glacier
|
|
external-doc-url: https://aws.amazon.com/glacier/
|
|
logo: /docs/integration-logos/Amazon-S3-Glacier_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/s3/glacier.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Kinesis Data Firehose
|
|
external-doc-url: https://aws.amazon.com/kinesis/data-firehose/
|
|
logo: /docs/integration-logos/Amazon-Kinesis-Data-Firehose_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Managed Service for Apache Flink
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst
|
|
external-doc-url: https://aws.amazon.com/managed-service-apache-flink/
|
|
logo: /docs/integration-logos/Amazon-Kinesis-Analytics_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon OpenSearch Serverless
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/opensearchserverless.rst
|
|
external-doc-url: https://aws.amazon.com/opensearchserverless/
|
|
logo: /docs/integration-logos/Amazon-OpenSearch_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon RDS
|
|
external-doc-url: https://aws.amazon.com/rds/
|
|
logo: /docs/integration-logos/Amazon-RDS_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/rds.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Redshift
|
|
external-doc-url: https://aws.amazon.com/redshift/
|
|
logo: /docs/integration-logos/Amazon-Redshift_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst
|
|
- /docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Redshift Data
|
|
external-doc-url: https://aws.amazon.com/redshift/
|
|
logo: /docs/integration-logos/Amazon-Redshift_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon SageMaker
|
|
external-doc-url: https://aws.amazon.com/sagemaker/
|
|
logo: /docs/integration-logos/Amazon-SageMaker_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/sagemaker.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon SageMaker Unified Studio
|
|
external-doc-url: https://aws.amazon.com/sagemaker/unified-studio/
|
|
logo: /docs/integration-logos/Amazon-SageMaker_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/sagemakerunifiedstudio.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon SecretsManager
|
|
external-doc-url: https://aws.amazon.com/secrets-manager/
|
|
logo: /docs/integration-logos/AWS-Secrets-Manager_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Simple Email Service (SES)
|
|
external-doc-url: https://aws.amazon.com/ses/
|
|
logo: /docs/integration-logos/Amazon-Simple-Email-Service-SES_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/ses.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Simple Notification Service (SNS)
|
|
external-doc-url: https://aws.amazon.com/sns/
|
|
logo: /docs/integration-logos/Amazon-Simple-Notification-Service-SNS_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/sns.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Simple Queue Service (SQS)
|
|
external-doc-url: https://aws.amazon.com/sqs/
|
|
logo: /docs/integration-logos/Amazon-Simple-Queue-Service-SQS_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/sqs.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
external-doc-url: https://aws.amazon.com/s3/
|
|
logo: /docs/integration-logos/Amazon-Simple-Storage-Service-S3_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/s3/s3.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon S3 Tables
|
|
external-doc-url: https://aws.amazon.com/s3/features/tables/
|
|
logo: /docs/integration-logos/Amazon-Simple-Storage-Service-S3_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/s3_tables.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Systems Manager (SSM)
|
|
external-doc-url: https://aws.amazon.com/systems-manager/
|
|
logo: /docs/integration-logos/AWS-Systems-Manager_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/ssm.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Web Services
|
|
external-doc-url: https://aws.amazon.com/
|
|
logo: /docs/integration-logos/AWS-Cloud-alt_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: AWS Batch
|
|
external-doc-url: https://aws.amazon.com/batch/
|
|
logo: /docs/integration-logos/AWS-Batch_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/batch.rst
|
|
tags: [aws]
|
|
- integration-name: AWS DataSync
|
|
external-doc-url: https://aws.amazon.com/datasync/
|
|
logo: /docs/integration-logos/AWS-DataSync_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: AWS Glue
|
|
external-doc-url: https://aws.amazon.com/glue/
|
|
logo: /docs/integration-logos/AWS-Glue_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/glue.rst
|
|
- /docs/apache-airflow-providers-amazon/operators/glue_catalog.rst
|
|
tags: [aws]
|
|
- integration-name: AWS Lambda
|
|
external-doc-url: https://aws.amazon.com/lambda/
|
|
logo: /docs/integration-logos/AWS-Lambda_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/lambda.rst
|
|
tags: [aws]
|
|
- integration-name: AWS Step Functions
|
|
external-doc-url: https://aws.amazon.com/step-functions/
|
|
logo: /docs/integration-logos/AWS-Step-Functions_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/step_functions.rst
|
|
tags: [aws]
|
|
- integration-name: AWS Database Migration Service
|
|
external-doc-url: https://aws.amazon.com/dms/
|
|
logo: /docs/integration-logos/AWS-Database-Migration-Service_64@5x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/dms.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon QuickSight
|
|
external-doc-url: https://aws.amazon.com/quicksight/
|
|
logo: /docs/integration-logos/Amazon-Quicksight_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/quicksight.rst
|
|
tags: [aws]
|
|
- integration-name: AWS Security Token Service (STS)
|
|
external-doc-url: https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html
|
|
logo: /docs/integration-logos/AWS-STS_light-bg@4x.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Appflow
|
|
external-doc-url: https://docs.aws.amazon.com/appflow/1.0/APIReference/Welcome.html
|
|
logo: /docs/integration-logos/Amazon_AppFlow_light.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/appflow.rst
|
|
tags: [aws]
|
|
- integration-name: AWS Glue DataBrew
|
|
external-doc-url: https://docs.aws.amazon.com/databrew/latest/dg/what-is.html
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/glue_databrew.rst
|
|
logo: /docs/integration-logos/AWS-Glue-DataBrew_64.png
|
|
tags: [aws]
|
|
- integration-name: Amazon Verified Permissions
|
|
external-doc-url: https://aws.amazon.com/verified-permissions/
|
|
logo: /docs/integration-logos/Amazon-Verified-Permissions.png
|
|
tags: [aws]
|
|
- integration-name: Amazon MWAA Serverless
|
|
external-doc-url: https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-serverless.html
|
|
logo: /docs/integration-logos/Amazon-MWAA.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/mwaa_serverless.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Neptune
|
|
external-doc-url: https://aws.amazon.com/neptune/
|
|
logo: /docs/integration-logos/Amazon-Neptune_64.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/neptune.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Kinesis Data Stream
|
|
external-doc-url: https://aws.amazon.com/kinesis/
|
|
logo: /docs/integration-logos/Amazon-Kinesis-Data-Firehose_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Managed Workflows for Apache Airflow (MWAA)
|
|
external-doc-url: https://aws.amazon.com/managed-workflows-for-apache-airflow/
|
|
logo: /docs/integration-logos/Amazon-MWAA.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/mwaa.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon Neptune Analytics
|
|
external-doc-url: https://docs.aws.amazon.com/neptune-analytics/latest/userguide/gettingStarted.html
|
|
logo: /docs/integration-logos/Amazon-Neptune_64.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/neptune_analytics.rst
|
|
tags: [aws]
|
|
- integration-name: Amazon MSK
|
|
external-doc-url: https://aws.amazon.com/msk/
|
|
tags: [aws]
|
|
- integration-name: Amazon S3 Vectors
|
|
external-doc-url: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vectors.html
|
|
logo: /docs/integration-logos/Amazon-Simple-Storage-Service-S3_light-bg@4x.png
|
|
how-to-guide:
|
|
- /docs/apache-airflow-providers-amazon/operators/s3_vectors.rst
|
|
tags: [aws]
|
|
|
|
|
|
operators:
|
|
- integration-name: Amazon Athena
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.athena
|
|
- airflow.providers.amazon.aws.operators.athena_spark
|
|
- integration-name: Amazon Web Services
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.base_aws
|
|
- integration-name: AWS Batch
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.batch
|
|
- integration-name: Amazon Bedrock
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.bedrock
|
|
- integration-name: Amazon CloudFormation
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.cloud_formation
|
|
- integration-name: Amazon Comprehend
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.comprehend
|
|
- integration-name: Amazon DataSync
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.datasync
|
|
- integration-name: AWS Database Migration Service
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.dms
|
|
- integration-name: Amazon EC2
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.ec2
|
|
- integration-name: Amazon Elastic Container Registry (ECR)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.ecr
|
|
- integration-name: Amazon ECS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.ecs
|
|
- integration-name: Amazon Elastic Kubernetes Service (EKS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.eks
|
|
- integration-name: Amazon EMR
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.emr
|
|
- integration-name: Amazon EMR on EKS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.emr
|
|
- integration-name: Amazon EventBridge
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.eventbridge
|
|
- integration-name: Amazon Glacier
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.glacier
|
|
- integration-name: AWS Glue
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.glue
|
|
- airflow.providers.amazon.aws.operators.glue_catalog
|
|
- airflow.providers.amazon.aws.operators.glue_crawler
|
|
- integration-name: AWS Lambda
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.lambda_function
|
|
- integration-name: Amazon Managed Service for Apache Flink
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.kinesis_analytics
|
|
- integration-name: Amazon Managed Workflows for Apache Airflow (MWAA)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.mwaa
|
|
- integration-name: Amazon MWAA Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.mwaa_serverless
|
|
- integration-name: Amazon OpenSearch Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.opensearch_serverless
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.s3
|
|
- integration-name: Amazon S3 Tables
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.s3_tables
|
|
- integration-name: Amazon SageMaker
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.sagemaker
|
|
- integration-name: Amazon SageMaker Unified Studio
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.sagemaker_unified_studio
|
|
- airflow.providers.amazon.aws.operators.sagemaker_unified_studio_notebook
|
|
- integration-name: Amazon Simple Email Service (SES)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.ses
|
|
- integration-name: Amazon Simple Notification Service (SNS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.sns
|
|
- integration-name: Amazon Simple Queue Service (SQS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.sqs
|
|
- integration-name: AWS Step Functions
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.step_function
|
|
- integration-name: Amazon Systems Manager (SSM)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.ssm
|
|
- integration-name: Amazon RDS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.rds
|
|
- integration-name: Amazon Redshift
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.redshift_cluster
|
|
- airflow.providers.amazon.aws.operators.redshift_data
|
|
- integration-name: Amazon QuickSight
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.quicksight
|
|
- integration-name: Amazon Appflow
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.appflow
|
|
- integration-name: AWS Glue DataBrew
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.glue_databrew
|
|
- integration-name: Amazon Neptune
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.neptune
|
|
- airflow.providers.amazon.aws.operators.neptune_analytics
|
|
- integration-name: Amazon S3 Vectors
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.operators.s3_vectors
|
|
|
|
sensors:
|
|
- integration-name: Amazon Athena
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.athena
|
|
- airflow.providers.amazon.aws.sensors.athena_spark
|
|
- integration-name: Amazon Web Services
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.base_aws
|
|
- integration-name: AWS Batch
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.batch
|
|
- integration-name: Amazon Bedrock
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.bedrock
|
|
- integration-name: Amazon CloudFormation
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.cloud_formation
|
|
- integration-name: Amazon Comprehend
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.comprehend
|
|
- integration-name: AWS Database Migration Service
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.dms
|
|
- integration-name: Amazon DynamoDB
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.dynamodb
|
|
- integration-name: Amazon EC2
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.ec2
|
|
- integration-name: Amazon ECS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.ecs
|
|
- integration-name: Amazon Elastic Kubernetes Service (EKS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.eks
|
|
- integration-name: Amazon EMR
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.emr
|
|
- integration-name: Amazon EMR on EKS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.emr
|
|
- integration-name: Amazon Glacier
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.glacier
|
|
- integration-name: AWS Glue
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.glue
|
|
- airflow.providers.amazon.aws.sensors.glue_crawler
|
|
- airflow.providers.amazon.aws.sensors.glue_catalog_partition
|
|
- integration-name: AWS Lambda
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.lambda_function
|
|
- integration-name: Amazon Managed Service for Apache Flink
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.kinesis_analytics
|
|
- integration-name: Amazon Managed Workflows for Apache Airflow (MWAA)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.mwaa
|
|
- integration-name: Amazon MWAA Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.mwaa_serverless
|
|
- integration-name: Amazon OpenSearch Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.opensearch_serverless
|
|
- integration-name: Amazon RDS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.rds
|
|
- integration-name: Amazon Redshift
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.redshift_cluster
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.s3
|
|
- integration-name: Amazon SageMaker
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.sagemaker
|
|
- integration-name: Amazon SageMaker Unified Studio
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.sagemaker_unified_studio
|
|
- airflow.providers.amazon.aws.sensors.sagemaker_unified_studio_notebook
|
|
- integration-name: Amazon Simple Queue Service (SQS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.sqs
|
|
- integration-name: AWS Step Functions
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.step_function
|
|
- integration-name: Amazon Systems Manager (SSM)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.ssm
|
|
- integration-name: Amazon QuickSight
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.sensors.quicksight
|
|
|
|
asset-uris:
|
|
- schemes: [s3]
|
|
handler: airflow.providers.amazon.aws.assets.s3.sanitize_uri
|
|
to_openlineage_converter: airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage
|
|
factory: airflow.providers.amazon.aws.assets.s3.create_asset
|
|
- schemes: [redshift]
|
|
handler: airflow.providers.amazon.aws.assets.redshift.sanitize_uri
|
|
factory: airflow.providers.amazon.aws.assets.redshift.create_asset
|
|
to_openlineage_converter: airflow.providers.amazon.aws.assets.redshift.convert_asset_to_openlineage
|
|
|
|
# dataset has been renamed to asset in Airflow 3.0
|
|
# This is kept for backward compatibility.
|
|
dataset-uris:
|
|
- schemes: [s3]
|
|
handler: airflow.providers.amazon.aws.assets.s3.sanitize_uri
|
|
to_openlineage_converter: airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage
|
|
factory: airflow.providers.amazon.aws.assets.s3.create_asset
|
|
- schemes: [redshift]
|
|
handler: airflow.providers.amazon.aws.assets.redshift.sanitize_uri
|
|
factory: airflow.providers.amazon.aws.assets.redshift.create_asset
|
|
to_openlineage_converter: airflow.providers.amazon.aws.assets.redshift.convert_asset_to_openlineage
|
|
|
|
filesystems:
|
|
- airflow.providers.amazon.aws.fs.s3
|
|
|
|
hooks:
|
|
- integration-name: Amazon Athena
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.athena
|
|
- airflow.providers.amazon.aws.hooks.athena_sql
|
|
- integration-name: DuckDB on AWS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.duckdb
|
|
- integration-name: Amazon Bedrock
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.bedrock
|
|
- integration-name: Amazon Chime
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.chime
|
|
- integration-name: Amazon Comprehend
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.comprehend
|
|
- integration-name: Amazon DynamoDB
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.dynamodb
|
|
- integration-name: Amazon Web Services
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.base_aws
|
|
- integration-name: AWS Batch
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.batch_client
|
|
- airflow.providers.amazon.aws.hooks.batch_waiters
|
|
- integration-name: Amazon CloudFormation
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.cloud_formation
|
|
- integration-name: Amazon DataSync
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.datasync
|
|
- integration-name: AWS Database Migration Service
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.dms
|
|
- integration-name: Amazon EC2
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.ec2
|
|
- integration-name: Amazon Elastic Container Registry (ECR)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.ecr
|
|
- integration-name: Amazon ECS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.ecs
|
|
- integration-name: Amazon ElastiCache
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.elasticache_replication_group
|
|
- integration-name: Amazon Elastic Kubernetes Service (EKS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.eks
|
|
- integration-name: Amazon EMR
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.emr
|
|
- integration-name: Amazon EMR on EKS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.emr
|
|
- integration-name: Amazon EventBridge
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.eventbridge
|
|
- integration-name: Amazon Glacier
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.glacier
|
|
- integration-name: AWS Glue
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.glue
|
|
- airflow.providers.amazon.aws.hooks.glue_crawler
|
|
- airflow.providers.amazon.aws.hooks.glue_catalog
|
|
- integration-name: Amazon Kinesis Data Stream
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.kinesis
|
|
- integration-name: Amazon Kinesis Data Firehose
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.firehose
|
|
- integration-name: AWS Lambda
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.lambda_function
|
|
- integration-name: Amazon Managed Service for Apache Flink
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.kinesis_analytics
|
|
- integration-name: Amazon CloudWatch Logs
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.logs
|
|
- integration-name: Amazon Managed Workflows for Apache Airflow (MWAA)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.mwaa
|
|
- integration-name: Amazon OpenSearch Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.opensearch_serverless
|
|
- integration-name: Amazon RDS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.rds
|
|
- integration-name: Amazon Redshift
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.redshift_sql
|
|
- airflow.providers.amazon.aws.hooks.redshift_cluster
|
|
- airflow.providers.amazon.aws.hooks.redshift_data
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.s3
|
|
- integration-name: Amazon S3 Tables
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.s3_tables
|
|
- integration-name: Amazon SageMaker
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.sagemaker
|
|
- integration-name: Amazon SageMaker Unified Studio
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.sagemaker_unified_studio
|
|
- airflow.providers.amazon.aws.hooks.sagemaker_unified_studio_notebook
|
|
- integration-name: Amazon Simple Email Service (SES)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.ses
|
|
- integration-name: Amazon Systems Manager (SSM)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.ssm
|
|
- integration-name: Amazon SecretsManager
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.secrets_manager
|
|
- integration-name: Amazon Simple Notification Service (SNS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.sns
|
|
- integration-name: Amazon Simple Queue Service (SQS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.sqs
|
|
- integration-name: AWS Step Functions
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.step_function
|
|
- integration-name: Amazon QuickSight
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.quicksight
|
|
- integration-name: AWS Security Token Service (STS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.sts
|
|
- integration-name: Amazon Appflow
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.appflow
|
|
- integration-name: AWS Glue DataBrew
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.glue_databrew
|
|
- integration-name: Amazon Verified Permissions
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.verified_permissions
|
|
- integration-name: Amazon Neptune
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.neptune
|
|
- airflow.providers.amazon.aws.hooks.neptune_analytics
|
|
- integration-name: Amazon MSK
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.hooks.msk
|
|
|
|
bundles:
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.bundles.s3
|
|
|
|
triggers:
|
|
- integration-name: Amazon Web Services
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.base
|
|
- integration-name: Amazon Athena
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.athena
|
|
- integration-name: AWS Batch
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.batch
|
|
- integration-name: Amazon Bedrock
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.bedrock
|
|
- integration-name: Amazon Comprehend
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.comprehend
|
|
- integration-name: Amazon DynamoDB
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.dynamodb
|
|
- integration-name: Amazon EC2
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.ec2
|
|
- integration-name: AWS Lambda
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.lambda_function
|
|
- integration-name: Amazon Kinesis Data Stream
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.kinesis
|
|
- integration-name: Amazon Managed Workflows for Apache Airflow (MWAA)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.mwaa
|
|
- integration-name: Amazon Managed Service for Apache Flink
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.kinesis_analytics
|
|
- integration-name: Amazon OpenSearch Serverless
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.opensearch_serverless
|
|
- integration-name: Amazon Redshift
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.redshift_cluster
|
|
- airflow.providers.amazon.aws.triggers.redshift_data
|
|
- integration-name: Amazon SageMaker
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.sagemaker
|
|
- integration-name: Amazon SageMaker Unified Studio
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.sagemaker_unified_studio
|
|
- airflow.providers.amazon.aws.triggers.sagemaker_unified_studio_notebook
|
|
- integration-name: AWS Glue
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.glue
|
|
- airflow.providers.amazon.aws.triggers.glue_crawler
|
|
- integration-name: Amazon Simple Storage Service (S3)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.s3
|
|
- integration-name: Amazon Systems Manager (SSM)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.ssm
|
|
- integration-name: Amazon EMR
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.emr
|
|
- integration-name: Amazon Elastic Kubernetes Service (EKS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.eks
|
|
- integration-name: Amazon ECS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.ecs
|
|
- integration-name: Amazon RDS
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.rds
|
|
- integration-name: AWS Step Functions
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.step_function
|
|
- integration-name: Amazon Simple Queue Service (SQS)
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.sqs
|
|
- integration-name: AWS Glue DataBrew
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.glue_databrew
|
|
- integration-name: Amazon Neptune
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.neptune
|
|
- airflow.providers.amazon.aws.triggers.neptune_analytics
|
|
- integration-name: AWS Database Migration Service
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.dms
|
|
- integration-name: Amazon QuickSight
|
|
python-modules:
|
|
- airflow.providers.amazon.aws.triggers.quicksight
|
|
|
|
transfers:
|
|
- source-integration-name: Amazon DynamoDB
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3
|
|
- source-integration-name: Google Cloud Storage (GCS)
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3
|
|
- source-integration-name: Amazon Glacier
|
|
target-integration-name: Google Cloud Storage (GCS)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.glacier_to_gcs
|
|
- source-integration-name: Google
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.google_api_to_s3
|
|
- source-integration-name: Apache Hive
|
|
target-integration-name: Amazon DynamoDB
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.hive_to_dynamodb
|
|
- source-integration-name: Hypertext Transfer Protocol (HTTP)
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/http_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.http_to_s3
|
|
- source-integration-name: Internet Message Access Protocol (IMAP)
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.imap_attachment_to_s3
|
|
- source-integration-name: MongoDB
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3
|
|
- source-integration-name: Amazon Redshift
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3
|
|
- source-integration-name: Amazon Simple Storage Service (S3)
|
|
target-integration-name: Amazon Redshift
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift
|
|
- source-integration-name: Amazon Simple Storage Service (S3)
|
|
target-integration-name: SSH File Transfer Protocol (SFTP)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp
|
|
- source-integration-name: SSH File Transfer Protocol (SFTP)
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3
|
|
- source-integration-name: Amazon Simple Storage Service (S3)
|
|
target-integration-name: File Transfer Protocol (FTP)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.s3_to_ftp
|
|
- source-integration-name: Exasol
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/exasol_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.exasol_to_s3
|
|
- source-integration-name: File Transfer Protocol (FTP)
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.ftp_to_s3
|
|
- source-integration-name: Salesforce
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.salesforce_to_s3
|
|
- source-integration-name: Local
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.local_to_s3
|
|
- source-integration-name: Common SQL
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.sql_to_s3
|
|
- source-integration-name: Amazon Simple Storage Service (S3)
|
|
target-integration-name: Common SQL
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.s3_to_sql
|
|
- source-integration-name: Amazon Web Services
|
|
target-integration-name: Amazon Web Services
|
|
python-module: airflow.providers.amazon.aws.transfers.base
|
|
- source-integration-name: Microsoft Azure Blob Storage
|
|
target-integration-name: Amazon Simple Storage Service (S3)
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/azure_blob_to_s3.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.azure_blob_to_s3
|
|
- source-integration-name: Amazon Simple Storage Service (S3)
|
|
target-integration-name: Amazon DynamoDB
|
|
how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_dynamodb.rst
|
|
python-module: airflow.providers.amazon.aws.transfers.s3_to_dynamodb
|
|
|
|
extra-links:
|
|
- airflow.providers.amazon.aws.links.athena.AthenaQueryResultsLink
|
|
- airflow.providers.amazon.aws.links.batch.BatchJobDefinitionLink
|
|
- airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink
|
|
- airflow.providers.amazon.aws.links.batch.BatchJobQueueLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrClusterLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrLogsLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrServerlessCloudWatchLogsLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrServerlessDashboardLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrServerlessLogsLink
|
|
- airflow.providers.amazon.aws.links.emr.EmrServerlessS3LogsLink
|
|
- airflow.providers.amazon.aws.links.glue.GlueJobRunDetailsLink
|
|
- airflow.providers.amazon.aws.links.logs.CloudWatchEventsLink
|
|
- airflow.providers.amazon.aws.links.sagemaker.SageMakerTransformJobLink
|
|
- airflow.providers.amazon.aws.links.sagemaker_unified_studio.SageMakerUnifiedStudioLink
|
|
- airflow.providers.amazon.aws.links.step_function.StateMachineDetailsLink
|
|
- airflow.providers.amazon.aws.links.step_function.StateMachineExecutionsDetailsLink
|
|
- airflow.providers.amazon.aws.links.comprehend.ComprehendPiiEntitiesDetectionLink
|
|
- airflow.providers.amazon.aws.links.comprehend.ComprehendDocumentClassifierLink
|
|
- airflow.providers.amazon.aws.links.datasync.DataSyncTaskLink
|
|
- airflow.providers.amazon.aws.links.datasync.DataSyncTaskExecutionLink
|
|
- airflow.providers.amazon.aws.links.ec2.EC2InstanceLink
|
|
- airflow.providers.amazon.aws.links.ec2.EC2InstanceDashboardLink
|
|
- airflow.providers.amazon.aws.links.neptune_analytics.NeptuneGraphLink
|
|
- airflow.providers.amazon.aws.links.neptune_analytics.NeptuneImportTaskLink
|
|
- airflow.providers.amazon.aws.links.ec2.VpcEndpointLink
|
|
|
|
connection-types:
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook
|
|
hook-name: "Amazon Web Services"
|
|
connection-type: aws
|
|
ui-field-behaviour:
|
|
hidden-fields:
|
|
- host
|
|
- schema
|
|
- port
|
|
relabeling:
|
|
login: AWS Access Key ID
|
|
password: AWS Secret Access Key
|
|
placeholders:
|
|
login: YOUR_AWS_ACCESS_KEY_ID
|
|
password: YOUR_AWS_SECRET_ACCESS_KEY
|
|
extra: |
|
|
{
|
|
"region_name": "us-east-1",
|
|
"session_kwargs": {"profile_name": "default"},
|
|
"config_kwargs": {"retries": {"mode": "standard", "max_attempts": 10}},
|
|
"role_arn": "arn:aws:iam::123456789098:role/role-name",
|
|
"assume_role_method": "assume_role",
|
|
"assume_role_kwargs": {"RoleSessionName": "airflow"},
|
|
"aws_session_token": "YOUR_AWS_SESSION_TOKEN",
|
|
"endpoint_url": "http://localhost:4566"
|
|
}
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook
|
|
hook-name: "Amazon Chime Webhook"
|
|
connection-type: chime
|
|
ui-field-behaviour:
|
|
hidden-fields:
|
|
- login
|
|
- port
|
|
- extra
|
|
relabeling:
|
|
host: Chime Webhook Endpoint
|
|
password: Chime Webhook token
|
|
placeholders:
|
|
schema: https
|
|
host: hooks.chime.aws/incomingwebhook/
|
|
password: T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.duckdb.AwsDuckDBHook
|
|
hook-name: "DuckDB on AWS"
|
|
connection-type: duckdb_aws
|
|
ui-field-behaviour:
|
|
hidden-fields:
|
|
- login
|
|
- password
|
|
- port
|
|
- schema
|
|
relabeling:
|
|
host: Database path
|
|
placeholders:
|
|
host: /tmp/analytics.duckdb (leave empty for an in-memory database)
|
|
extra: |
|
|
{
|
|
"credential_strategy": "credential_chain",
|
|
"region_name": "us-east-1",
|
|
"memory_limit": "2GB",
|
|
"threads": 4
|
|
}
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook
|
|
hook-name: "Amazon Elastic MapReduce"
|
|
connection-type: emr
|
|
ui-field-behaviour:
|
|
hidden-fields:
|
|
- host
|
|
- schema
|
|
- port
|
|
- login
|
|
- password
|
|
relabeling:
|
|
extra: Run Job Flow Configuration
|
|
placeholders:
|
|
extra: |
|
|
{
|
|
"Name": "MyClusterName",
|
|
"ReleaseLabel": "emr-5.36.0",
|
|
"Applications": [{"Name": "Spark"}],
|
|
"Instances": {
|
|
"InstanceGroups": [{
|
|
"Name": "Primary node",
|
|
"Market": "SPOT",
|
|
"InstanceRole": "MASTER",
|
|
"InstanceType": "m5.large",
|
|
"InstanceCount": 1
|
|
}],
|
|
"KeepJobFlowAliveWhenNoSteps": false,
|
|
"TerminationProtected": false
|
|
},
|
|
"StepConcurrencyLevel": 2
|
|
}
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook
|
|
hook-name: "Amazon Redshift"
|
|
connection-type: redshift
|
|
ui-field-behaviour:
|
|
relabeling:
|
|
login: User
|
|
schema: Database
|
|
- hook-class-name: airflow.providers.amazon.aws.hooks.athena_sql.AthenaSQLHook
|
|
hook-name: "Amazon Athena"
|
|
connection-type: athena
|
|
ui-field-behaviour:
|
|
hidden-fields:
|
|
- host
|
|
- port
|
|
relabeling:
|
|
login: AWS Access Key ID
|
|
password: AWS Secret Access Key
|
|
placeholders:
|
|
login: YOUR_AWS_ACCESS_KEY_ID
|
|
password: YOUR_AWS_SECRET_ACCESS_KEY
|
|
extra: |
|
|
{
|
|
"aws_domain": "amazonaws.com",
|
|
"driver": "rest",
|
|
"s3_staging_dir": "s3://bucket_name/staging/",
|
|
"work_group": "primary",
|
|
"region_name": "us-east-1",
|
|
"session_kwargs": {"profile_name": "default"},
|
|
"config_kwargs": {"retries": {"mode": "standard", "max_attempts": 10}},
|
|
"role_arn": "arn:aws:iam::123456789098:role/role-name",
|
|
"assume_role_method": "assume_role",
|
|
"assume_role_kwargs": {"RoleSessionName": "airflow"},
|
|
"aws_session_token": "YOUR_AWS_SESSION_TOKEN",
|
|
"endpoint_url": "http://localhost:4566"
|
|
}
|
|
|
|
notifications:
|
|
- airflow.providers.amazon.aws.notifications.chime.ChimeNotifier
|
|
- airflow.providers.amazon.aws.notifications.ses.SesNotifier
|
|
- airflow.providers.amazon.aws.notifications.sns.SnsNotifier
|
|
- airflow.providers.amazon.aws.notifications.sqs.SqsNotifier
|
|
|
|
secrets-backends:
|
|
- airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend
|
|
- airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
|
|
|
|
email-backends:
|
|
- airflow.providers.amazon.aws.utils.emailer.send_email
|
|
|
|
logging:
|
|
- airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler
|
|
- airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler
|
|
|
|
remote-logging:
|
|
- classpath: airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudWatchRemoteLogIO
|
|
scheme: cloudwatch
|
|
- classpath: airflow.providers.amazon.aws.log.s3_task_handler.S3RemoteLogIO
|
|
scheme: s3
|
|
|
|
config:
|
|
aws:
|
|
description: This section contains settings for Amazon Web Services (AWS) integration.
|
|
options:
|
|
session_factory:
|
|
description: |
|
|
Full import path to the class which implements a custom session factory for
|
|
``boto3.session.Session``. For more details please have a look at
|
|
:ref:`howto/connection:aws:session-factory`.
|
|
default: ~
|
|
example: my_company.aws.MyCustomSessionFactory
|
|
type: string
|
|
version_added: 3.1.1
|
|
cloudwatch_task_handler_json_serializer:
|
|
description: |
|
|
By default, when logging non-string messages, all non-json objects are logged as `null`.
|
|
Except `datetime` objects which are ISO formatted. Users can optionally use a `repr` serializer or
|
|
provide their own JSON serializer for any non-JSON-serializable objects in the logged message.
|
|
|
|
* `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize` uses `repr` (be aware
|
|
there is the potential of logging sensitive data depending on the `repr` method of logged objects)
|
|
* `airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy` uses `null`.
|
|
|
|
If a custom serializer is provided, it must adhere to `Callable[[Any], str | None]`, where `None`
|
|
serializes to `null` (e.g. `def my_serializer(o: Any) -> str | None`). Since this is on the logging
|
|
path and it's possible there's an exception being handled, special care should be taken to fail
|
|
gracefully without raising a new exception inside of your serializer.
|
|
type: string
|
|
version_added: 8.7.2
|
|
example: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize
|
|
default: airflow.providers.amazon.aws.log.cloudwatch_task_handler.json_serialize_legacy
|
|
s3_task_handler_acl_policy:
|
|
description: |
|
|
The ACL applied to task log objects uploaded to S3 by the S3 remote log handler,
|
|
for example ``bucket-owner-full-control``.
|
|
|
|
This is primarily useful for cross-account remote logging: when Airflow runs under one AWS
|
|
account but writes logs to a bucket owned by another account, S3 makes the writing account
|
|
the object owner, so the bucket owner cannot read or manage the log objects. Setting
|
|
``bucket-owner-full-control`` grants the bucket owner full control over the uploaded logs.
|
|
|
|
When unset, no ACL is sent and the bucket's default object ownership applies.
|
|
type: string
|
|
version_added: 9.34.0
|
|
example: bucket-owner-full-control
|
|
default: ~
|
|
aws_batch_executor:
|
|
description: |
|
|
This section only applies if you are using the AwsBatchExecutor in
|
|
Airflow's ``[core]`` configuration.
|
|
For more information on any of these execution parameters, see the link below:
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/batch.html#Batch.Client.submit_job
|
|
For boto3 credential management, see
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
|
|
options:
|
|
conn_id:
|
|
description: |
|
|
The Airflow connection (i.e. credentials) used by the Batch executor to make API calls to AWS Batch.
|
|
version_added: "8.11"
|
|
type: string
|
|
example: "aws_default"
|
|
default: "aws_default"
|
|
region_name:
|
|
description: |
|
|
The name of the AWS Region where Amazon Batch is configured. Required.
|
|
version_added: "8.11"
|
|
type: string
|
|
example: "us-east-1"
|
|
default: ~
|
|
max_submit_job_attempts:
|
|
description: |
|
|
The maximum number of times the Batch Executor should attempt to run a Batch Job.
|
|
version_added: "8.11"
|
|
type: integer
|
|
example: "3"
|
|
default: "3"
|
|
check_health_on_startup:
|
|
description: |
|
|
Whether or not to check the Batch Executor health on startup.
|
|
version_added: "8.11"
|
|
type: boolean
|
|
example: "True"
|
|
default: "True"
|
|
job_name:
|
|
description: |
|
|
The name of the job submitted to AWS Batch. It can be up to 128 letters long.
|
|
The first character must be alphanumeric, can contain uppercase and lowercase
|
|
letters, numbers, hyphens (-), and underscores (_).
|
|
version_added: "8.11"
|
|
type: string
|
|
example: "airflow-batch-executor-job"
|
|
default: ~
|
|
job_queue:
|
|
description: |
|
|
The job queue where the job is submitted. You can specify either the name
|
|
or the Amazon Resource Name (ARN) of the queue.
|
|
version_added: "8.11"
|
|
type: string
|
|
example: "airflow-batch-executor-job-queue"
|
|
default: ~
|
|
job_definition:
|
|
description: |
|
|
The job definition used by the job. You can specify either the name
|
|
or the Amazon Resource Name (ARN) of the job definition with or
|
|
without the revision. If the revision is not specified, then the
|
|
latest active revision is used.
|
|
version_added: "8.11"
|
|
type: string
|
|
example: "airflow-batch-executor-job-definition"
|
|
default: ~
|
|
submit_job_kwargs:
|
|
description: |
|
|
Additional parameters to pass to the submit_job method of the
|
|
AWS Batch client.
|
|
version_added: "8.11"
|
|
type: string
|
|
example: '{"Tags": [{"Key": "key", "Value": "value"}]}'
|
|
default: ~
|
|
|
|
aws_lambda_executor:
|
|
description: |
|
|
This section only applies if you are using the AwsLambdaExecutor in
|
|
Airflow's ``[core.executor]`` configuration.
|
|
For more information see:
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/lambda/client/invoke.html
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
|
|
https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/executors/lambda-executor.html
|
|
options:
|
|
conn_id:
|
|
description: |
|
|
The Airflow connection (i.e. credentials) used by the Lambda executor to make API calls.
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "aws_default"
|
|
default: "aws_default"
|
|
region_name:
|
|
description: |
|
|
The name of the AWS Region where Amazon Lambda is configured.
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "us-east-1"
|
|
default: ~
|
|
check_health_on_startup:
|
|
description: |
|
|
Whether or not to check the Lambda Executor health on startup.
|
|
version_added: "9.9.0"
|
|
type: boolean
|
|
example: "True"
|
|
default: "True"
|
|
max_invoke_attempts:
|
|
description: |
|
|
The maximum number of times the Lambda Executor should attempt to start an Airflow task.
|
|
version_added: "9.9.0"
|
|
type: integer
|
|
example: "3"
|
|
default: "3"
|
|
queue_url:
|
|
description: |
|
|
The URL of the SQS queue to use for the Lambda executor. Required.
|
|
The Lambda executor will poll this queue for results of the lambda function's Airflow Task.
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "airflow-lambda-executor-results-queue"
|
|
default: ~
|
|
dead_letter_queue_url:
|
|
description: |
|
|
The URL of the SQS dead letter queue to use for the Lambda function. The Lambda executor
|
|
will poll this queue for timeout/exception results of the lambda function. Required
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "airflow-lambda-executor-dlq"
|
|
default: ~
|
|
function_name:
|
|
description: |
|
|
The name of the Lambda function to invoke. Required.
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "airflow-lambda-executor-function"
|
|
default: ~
|
|
qualifier:
|
|
description: |
|
|
The version or alias of the Lambda function to invoke. If not specified, the latest version is used.
|
|
version_added: "9.9.0"
|
|
type: string
|
|
example: "1"
|
|
default: ~
|
|
end_wait_timeout:
|
|
description: |
|
|
The number of seconds to wait for all lambda executor invocations to complete when terminating the
|
|
executor/scheduler. Default of 0 means wait indefinitely.
|
|
version_added: "9.9.0"
|
|
type: integer
|
|
example: "500"
|
|
default: "0"
|
|
|
|
aws_ecs_executor:
|
|
description: |
|
|
This section only applies if you are using the AwsEcsExecutor in
|
|
Airflow's ``[core]`` configuration.
|
|
For more information on any of these execution parameters, see the link below:
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs/client/run_task.html
|
|
For boto3 credential management, see
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
|
|
options:
|
|
conn_id:
|
|
description: |
|
|
The Airflow connection (i.e. credentials) used by the ECS executor to make API calls to AWS ECS.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "aws_default"
|
|
default: "aws_default"
|
|
region_name:
|
|
description: |
|
|
The name of the AWS Region where Amazon ECS is configured. Required.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "us-east-1"
|
|
default: ~
|
|
assign_public_ip:
|
|
description: |
|
|
Whether to assign a public IP address to the containers launched by the ECS executor.
|
|
For more info see url to Boto3 docs above.
|
|
version_added: "8.10"
|
|
type: boolean
|
|
example: "True"
|
|
default: "False"
|
|
cluster:
|
|
description: |
|
|
Name of the Amazon ECS Cluster. Required.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "ecs_executor_cluster"
|
|
default: ~
|
|
capacity_provider_strategy:
|
|
description: |
|
|
The capacity provider strategy to use for the task.
|
|
|
|
If a Capacity Provider Strategy is specified, the Launch Type parameter must be omitted. If
|
|
no Capacity Provider Strategy or Launch Type is specified, the Default CapacityProvider Strategy
|
|
for the cluster is used, if present.
|
|
|
|
When you use cluster auto scaling, you must specify Capacity Provider Strategy and not Launch Type.
|
|
version_added: "8.17"
|
|
type: string
|
|
example: "[{'capacityProvider': 'cp1', 'weight': 5}, {'capacityProvider': 'cp2', 'weight': 1}]"
|
|
default: ~
|
|
container_name:
|
|
description: |
|
|
Name of the container that will be used to execute Airflow tasks via the ECS executor.
|
|
The container should be specified in the ECS Task Definition and will receive an airflow
|
|
CLI command as an additional parameter to its entrypoint. For more info see url to Boto3
|
|
docs above. Required.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "ecs_executor_container"
|
|
default: ~
|
|
launch_type:
|
|
description: |
|
|
Launch type can either be 'FARGATE' OR 'EC2'. For more info see url to
|
|
Boto3 docs above.
|
|
|
|
If a Launch Type is specified, the Capacity Provider Strategy parameter must be omitted. If
|
|
no Capacity Provider Strategy or Launch Type is specified, the Default Capacity Provider Strategy
|
|
for the cluster is used, if present.
|
|
|
|
If the launch type is EC2, the executor will attempt to place tasks on
|
|
empty EC2 instances. If there are no EC2 instances available, no task
|
|
is placed and this function will be called again in the next heart-beat.
|
|
|
|
If the launch type is FARGATE, this will run the tasks on new AWS Fargate
|
|
instances.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "FARGATE"
|
|
default: ~
|
|
platform_version:
|
|
description: |
|
|
The platform version the task uses. A platform version is only specified
|
|
for tasks hosted on Fargate. If one isn't specified, the LATEST platform
|
|
version is used.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "1.4.0"
|
|
default: "LATEST"
|
|
security_groups:
|
|
description: |
|
|
The comma-separated IDs of the security groups associated with the task. If you
|
|
don't specify a security group, the default security group for the VPC is used.
|
|
There's a limit of 5 security groups. For more info see url to Boto3 docs above.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "sg-XXXX,sg-YYYY"
|
|
default: ~
|
|
subnets:
|
|
description: |
|
|
The comma-separated IDs of the subnets associated with the task or service.
|
|
There's a limit of 16 subnets. For more info see url to Boto3 docs above.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "subnet-XXXXXXXX,subnet-YYYYYYYY"
|
|
default: ~
|
|
task_definition:
|
|
description: |
|
|
The family and revision (family:revision) or full ARN of the task definition
|
|
to run. If a revision isn't specified, the latest ACTIVE revision is used.
|
|
For more info see url to Boto3 docs above.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: executor_task_definition:LATEST
|
|
default: ~
|
|
max_run_task_attempts:
|
|
description: |
|
|
The maximum number of times the ECS Executor should attempt to run a task.
|
|
version_added: "8.10"
|
|
type: integer
|
|
example: "3"
|
|
default: "3"
|
|
run_task_concurrency:
|
|
description: |
|
|
The maximum number of ECS RunTask calls the ECS Executor makes in parallel. With 1, tasks are
|
|
launched one at a time and the scheduler loop waits for each call. With more, the calls run in
|
|
background threads and the scheduler loop does not wait for them.
|
|
version_added: "9.40.0"
|
|
type: integer
|
|
example: "16"
|
|
default: "1"
|
|
run_task_kwargs:
|
|
description: |
|
|
A JSON string containing arguments to provide the ECS `run_task` API (see url above).
|
|
version_added: "8.10"
|
|
type: string
|
|
example: '{"tags": {"key": "schema", "value": "1.0"}}'
|
|
default: ~
|
|
check_health_on_startup:
|
|
description: |
|
|
Whether or not to check the ECS Executor health on startup.
|
|
version_added: "8.11"
|
|
type: boolean
|
|
example: "True"
|
|
default: "True"
|
|
aws_eks_executor:
|
|
description: |
|
|
This section only applies if you are using the AwsEksExecutor in
|
|
Airflow's ``[core]`` configuration. It requires the ``cncf.kubernetes``
|
|
extra of this provider. The executor supplies the Kubernetes client for
|
|
the configured Amazon EKS cluster; all pod-level settings are still read
|
|
from the ``[kubernetes_executor]`` section.
|
|
For boto3 credential management, see
|
|
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html
|
|
options:
|
|
conn_id:
|
|
description: |
|
|
The Airflow connection (i.e. credentials) used by the EKS executor to
|
|
make API calls to Amazon EKS.
|
|
version_added: 9.39.0
|
|
type: string
|
|
example: "aws_default"
|
|
default: "aws_default"
|
|
region_name:
|
|
description: |
|
|
The name of the AWS Region where the EKS cluster is located. If not
|
|
specified, falls back to the AWS connection's region, then boto3's.
|
|
version_added: 9.39.0
|
|
type: string
|
|
example: "us-east-1"
|
|
default: ~
|
|
check_health_on_startup:
|
|
description: |
|
|
Whether or not to check the EKS Executor health on startup.
|
|
version_added: 9.39.0
|
|
type: boolean
|
|
example: "True"
|
|
default: "True"
|
|
cluster_name:
|
|
description: |
|
|
The name of the Amazon EKS cluster to run Airflow tasks on. Required.
|
|
version_added: 9.39.0
|
|
type: string
|
|
example: "airflow-eks-cluster"
|
|
default: ~
|
|
aws_auth_manager:
|
|
description: |
|
|
This section only applies if you are using the AwsAuthManager. In other words, if you set
|
|
``[core] auth_manager = airflow.providers.amazon.aws.auth_manager.aws_auth_manager.AwsAuthManager`` in
|
|
Airflow's configuration.
|
|
options:
|
|
conn_id:
|
|
description: |
|
|
The Airflow connection (i.e. credentials) used by the AWS auth manager to make API calls to AWS
|
|
Identity Center and Amazon Verified Permissions.
|
|
version_added: 8.12.0
|
|
type: string
|
|
example: "aws_default"
|
|
default: "aws_default"
|
|
region_name:
|
|
description: |
|
|
The name of the AWS Region where Amazon Verified Permissions is configured. Required.
|
|
version_added: "8.10"
|
|
type: string
|
|
example: "us-east-1"
|
|
default: ~
|
|
saml_metadata_url:
|
|
description: |
|
|
SAML metadata XML file provided by AWS Identity Center.
|
|
This URL can be found in the AWS Identity Center console. Required.
|
|
version_added: 8.12.0
|
|
type: string
|
|
example: "https://portal.sso.<region>.amazonaws.com/saml/metadata/XXXXXXXXXX"
|
|
default: ~
|
|
avp_policy_store_id:
|
|
description: |
|
|
Amazon Verified Permissions' policy store ID where all the policies defining user permissions
|
|
in Airflow are stored. Required.
|
|
version_added: 8.12.0
|
|
type: string
|
|
example: ~
|
|
default: ~
|
|
allow_idp_initiated_login:
|
|
description: |
|
|
Whether to accept SAML assertions that no login started from Airflow asked for, such as
|
|
the ones produced by clicking the Airflow tile in the AWS Identity Center access portal
|
|
(IdP-initiated SSO).
|
|
|
|
When this is disabled, a SAML response is only accepted if it answers an AuthnRequest that
|
|
the same browser started. That binding is what stops an assertion obtained elsewhere from
|
|
being accepted in another user's browser, which would sign that user in as the assertion's
|
|
subject. Enable it only where the access portal flow is required and that trade-off is
|
|
accepted.
|
|
version_added: 9.37.0
|
|
type: boolean
|
|
example: "True"
|
|
default: "False"
|
|
|
|
executors:
|
|
- airflow.providers.amazon.aws.executors.aws_lambda.lambda_executor.AwsLambdaExecutor
|
|
- airflow.providers.amazon.aws.executors.batch.batch_executor.AwsBatchExecutor
|
|
- airflow.providers.amazon.aws.executors.ecs.ecs_executor.AwsEcsExecutor
|
|
- airflow.providers.amazon.aws.executors.eks.eks_executor.AwsEksExecutor
|
|
|
|
auth-managers:
|
|
- airflow.providers.amazon.aws.auth_manager.aws_auth_manager.AwsAuthManager
|
|
|
|
cli:
|
|
- airflow.providers.amazon.aws.cli.definition.get_aws_cli_commands
|
|
|
|
queues:
|
|
- airflow.providers.amazon.aws.queues.kinesis.KinesisMessageQueueProvider
|
|
- airflow.providers.amazon.aws.queues.sqs.SqsMessageQueueProvider
|