mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
github-actions[bot]andJarek Potiuk
44e8ff3e44
* [main] CI: Upgrade important CI environment * Bump pypdf to 6.21.4 to fix high-severity advisories The CI environment upgrade resolved pypdf 6.21.3, which the dependency review gate rejects (GHSA-mfr5-77xh-9j7h, GHSA-cggf-277g-3q54, GHSA-5849-834g-hvw5). The fixed 6.21.4 is still inside the 4-day exclude-newer window, so add a temporary per-package cutoff for it. Generated-by: Claude Opus 5 * Support pydantic-ai 2.60+ and opensandbox 1.2 in common.ai provider pydantic-ai 2.60 records a durable streamed model request as the bare response rather than a response-plus-events wrapper, so replaying or tracking a streamed request in AirflowDurability failed to decode it. The same release answers a tool retry with the call's own ToolReturnPart (outcome "retried") instead of RetryPromptPart, which broke tests that told retries from results by part type. opensandbox 1.2 allows an egress rule with ports and no target. When the enforced policy did not match the requested one, sorting the allowed targets for the error message raised TypeError on that None, hiding the real reason the sandbox was refused. Both work with the older releases too, which the provider still supports. Generated-by: Claude Opus 5 * Fix provider tests and type checks after the dependency upgrade The tool-part predicates in the common.ai tests returned a plain bool, so mypy could not narrow the message part they matched and the providers type check failed. On Python 3.14 the upgrade moves snowflake-connector-python to 5.1 and snowflake-sqlalchemy to 2.1. connector 5.1's connect() no longer goes through SnowflakeConnection, so patching that class stopped intercepting the connection and the test tried a real login. snowflake-sqlalchemy 2.0.0 started percent-encoding "?" in the URL password, which the URI test pinned to the older, unencoded form. Older Python versions keep the earlier releases, so both tests have to accept either. Generated-by: Claude Opus 5 --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
368 lines
15 KiB
YAML
368 lines
15 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
---
|
|
default_stages: [pre-commit, pre-push]
|
|
minimum_prek_version: '0.3.4'
|
|
default_language_version:
|
|
python: python3
|
|
node: 22.19.0
|
|
golang: 1.24.0
|
|
repos:
|
|
- repo: local
|
|
hooks:
|
|
- id: check-deferrable-default
|
|
name: Check and fix default value of default_deferrable
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: python
|
|
entry: ../scripts/ci/prek/check_deferrable_default.py
|
|
pass_filenames: false
|
|
files: ^(.*/)?airflow/.*/(sensors|operators)/.*\.py$
|
|
- id: check-trigger-serialize-init
|
|
name: Check trigger __init__ and serialize() are in sync
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: Every trigger __init__ parameter must appear in its serialize() return dict
|
|
language: python
|
|
entry: ../scripts/ci/prek/check_trigger_serialize_init.py
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*/triggers/[^/]+\.py$
|
|
- id: check-deferred-hook-configuration
|
|
name: Check deferred AWS tasks pass their hook configuration
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: An operator that defers must hand region_name, verify and botocore_config to its trigger
|
|
language: python
|
|
entry: ../scripts/ci/prek/check_deferred_hook_configuration.py
|
|
pass_filenames: false
|
|
files: ^amazon/src/airflow/providers/amazon/aws/.*\.py$
|
|
- id: check-migration-patterns
|
|
name: Check migration files for anti-patterns and style (MIG001-MIG004)
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_migration_patterns.py
|
|
language: python
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*/migrations/versions/.*\.py$
|
|
additional_dependencies: [rich>=13.6.0]
|
|
require_serial: true
|
|
- id: update-providers-dependencies
|
|
name: Update dependencies for providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/update_providers_dependencies.py
|
|
language: python
|
|
always_run: true
|
|
pass_filenames: false
|
|
- id: validate-operators-init
|
|
name: Validate templated fields in operator __init__
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: Require plain assignment of templated fields and forbid reading their un-rendered values
|
|
language: python
|
|
entry: ../scripts/ci/prek/validate_operators_init.py
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
|
|
- id: check-conn-id-templated
|
|
name: Check connection ids accepted by operators are in template_fields
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: Every *conn_id argument an operator, sensor or notifier accepts must be templated
|
|
language: python
|
|
entry: ../scripts/ci/prek/check_conn_id_templated.py
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
additional_dependencies: [rich>=13.6.0]
|
|
require_serial: true
|
|
- id: update-providers-build-files
|
|
name: Update providers build files
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/update_providers_build_files.py
|
|
language: python
|
|
pass_filenames: true
|
|
files: |
|
|
(?x)
|
|
^[^\/]*/src/airflow/providers/[^\/]*/__init__\.py$|
|
|
^[^\/]*/[^\/]*/src/airflow/providers/[^\/]*/[^\/]*/__init__\.py$|
|
|
^.*/pyproject\.toml$|
|
|
^.*/provider\.yaml$|
|
|
^../dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE\.py\.jinja2$|
|
|
^../dev/breeze/src/airflow_breeze/templates/get_provider_info_TEMPLATE\.py\.jinja2$|
|
|
^../dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE\.rst\.jinja2$
|
|
require_serial: true
|
|
- id: check-airflow-v-imports-in-tests
|
|
name: Check AIRFLOW_V imports in tests
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: python
|
|
entry: ../scripts/ci/prek/check_airflow_v_imports_in_tests.py
|
|
pass_filenames: true
|
|
files: ^.*/tests/.+\.py$
|
|
- id: check-sql-dependency-common-data-structure
|
|
name: Check dependency of SQL providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: Check dependency of SQL Providers with common data structure
|
|
entry: ../scripts/ci/prek/check_common_sql_dependency.py
|
|
language: python
|
|
files: ^.*/src/airflow/providers/.*/hooks/.*\.py$
|
|
- id: generate-volumes-for-sources
|
|
name: Generate volumes for docker compose
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/generate_volumes_for_sources.py
|
|
language: python
|
|
files: ^.*/provider\.yaml$
|
|
pass_filenames: false
|
|
require_serial: true
|
|
- id: check-pytest-mark-db-test-in-providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: pygrep
|
|
name: Check pytest.mark.db_test use in providers
|
|
entry: pytest\.mark\.db_test
|
|
pass_filenames: true
|
|
# Here we should add providers that are already free from the pytest.mark.db_test
|
|
# and we want to keep them clean and only use non-db-tests
|
|
files: >
|
|
(?x)
|
|
^airbyte/.*\.py$|
|
|
^apache/beam/.*\.py$|
|
|
^apache/flink/.*\.py$|
|
|
^apache/iceberg/.*\.py$|
|
|
^apache/kafka/.*\.py$|
|
|
^apprise/.*\.py$|
|
|
^arangodb/.*\.py$|
|
|
^asana/.*\.py$|
|
|
^atlassian/jira/.*\.py$|
|
|
^cloudant/.*\.py$|
|
|
^cohere/.*\.py$|
|
|
^common/compat/.*\.py$|
|
|
^common/messaging/.*\.py$|
|
|
^datadog/.*\.py$|
|
|
^dingding/.*\.py$|
|
|
^discord/.*\.py$|
|
|
^exasol/.*\.py$|
|
|
^facebook/.*\.py$|
|
|
^ftp/.*\.py$|
|
|
^grpc/.*\.py$|
|
|
^hashicorp/.*\.py$|
|
|
^imap/.*\.py$|
|
|
^influxdb/.*\.py$|
|
|
^jdbc/.*\.py$|
|
|
^jenkins/.*\.py$|
|
|
^mongo/.*\.py$|
|
|
^microsoft/psrp/.*\.py$|
|
|
^microsoft/winrm/.*\.py$|
|
|
^neo4j/.*\.py$|
|
|
^odbc/.*\.py$|
|
|
^openai/.*\.py$|
|
|
^openfaas/.*\.py$|
|
|
^opsgenie/.*\.py$|
|
|
^oracle/.*\.py$|
|
|
^pagerduty/.*\.py$|
|
|
^pgvector/.*\.py$|
|
|
^pinecone/.*\.py$|
|
|
^postgres/.*\.py$|
|
|
^presto/.*\.py$|
|
|
^segment/.*\.py$|
|
|
^sendgrid/.*\.py$|
|
|
^singularity/.*\.py$|
|
|
^slack/.*\.py$|
|
|
^smtp/.*\.py$|
|
|
^tableau/.*\.py$|
|
|
^teradata/.*\.py$|
|
|
^trino/.*\.py$|
|
|
^vertica/.*\.py$|
|
|
^yandex/.*\.py$|
|
|
^zendesk/.*\.py$
|
|
- id: check-links-to-example-dags-do-not-use-hardcoded-versions
|
|
name: Verify no hard-coded version in example dags
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
description: The links to example dags should use |version| as version specification
|
|
language: pygrep
|
|
entry: >
|
|
(?i)
|
|
.*https://github.*/main/providers/.*/src/airflow/providers/.*/example_dags/|
|
|
.*https://github.*/master/providers/.*/src/airflow/providers/.*/example_dags/
|
|
pass_filenames: true
|
|
files: ^.*/docs/.*\.rst
|
|
- id: check-no-airflow-deprecation-in-providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: pygrep
|
|
name: Do not use DeprecationWarning in providers
|
|
description: Use AirflowProviderDeprecationWarning in providers
|
|
entry: "^\\s*DeprecationWarning*"
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
- id: check-base-operator-usage
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: pygrep
|
|
name: Check BaseOperator other imports
|
|
description: Make sure BaseOperator is imported from airflow.models outside of core
|
|
entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
|
|
pass_filenames: true
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
exclude: ^standard/.*/.*\.py$
|
|
- id: check-get-lineage-collector-providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
language: python
|
|
name: Check providers import hook lineage code from compat
|
|
description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of
|
|
airflow.lineage.hook.
|
|
entry: ../scripts/ci/prek/check_airflow_imports.py
|
|
--pattern '^airflow\.lineage\.hook'
|
|
--message "Only TYPE_CHECKING imports from `airflow.lineage.hook` are allowed in providers."
|
|
--only_top_level
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
exclude: ^common/compat/src/airflow/providers/common/compat/.*\.py$
|
|
- id: check-aiobotocore-optional
|
|
name: Check if aiobotocore is an optional dependency only
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_aiobotocore_optional.py
|
|
language: python
|
|
files: ^.*/provider\.yaml$
|
|
pass_filenames: true
|
|
require_serial: true
|
|
- id: check-providers-subpackages-init-file-exist
|
|
name: Provider subpackage init files are there
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
pass_filenames: false
|
|
always_run: true
|
|
entry: ../scripts/ci/prek/check_providers_subpackages_all_have_init.py
|
|
language: python
|
|
require_serial: true
|
|
- id: check-system-tests-tocs
|
|
name: Check that system tests is properly added
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_system_tests_hidden_in_index.py
|
|
language: python
|
|
pass_filenames: true
|
|
files: ^.*/docs/index\.rst$
|
|
- id: check-provider-docs-valid
|
|
name: Validate provider doc files
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_provider_docs.py
|
|
language: python
|
|
files: ^.*/provider\.yaml$|^.*/docs/.*
|
|
require_serial: true
|
|
pass_filenames: false
|
|
- id: check-connection-doc-labels
|
|
name: Validate connection doc labels match provider.yaml
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_connection_doc_labels.py
|
|
language: python
|
|
files: ^.*/provider\.yaml$|^.*/docs/.*/connections.*\.rst$|^.*/docs/connections\.rst$
|
|
require_serial: true
|
|
pass_filenames: false
|
|
- id: check-common-ai-operators-index
|
|
name: Check common.ai operators are listed in the docs index
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_common_ai_operators_index.py
|
|
language: python
|
|
files: ^common/ai/(src/airflow/providers/common/ai/operators/.*\.py|docs/operators/index\.rst)$
|
|
require_serial: true
|
|
pass_filenames: false
|
|
- id: mypy-providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
stages: ['pre-push']
|
|
name: Run mypy for providers
|
|
language: python
|
|
entry: ../scripts/ci/prek/mypy.py
|
|
files: ^.*\.py$
|
|
require_serial: true
|
|
- id: mypy-providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
stages: ['manual']
|
|
name: Run mypy for providers (manual)
|
|
language: python
|
|
entry: ../scripts/ci/prek/mypy_folder.py providers
|
|
pass_filenames: false
|
|
files: ^.*\.py$
|
|
require_serial: true
|
|
- id: check-provider-yaml-valid
|
|
name: Validate provider.yaml files
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_provider_yaml_files.py
|
|
language: python
|
|
files: >
|
|
(?x)
|
|
^.*/provider\.yaml$|
|
|
^.*/src/airflow/providers/.*/hooks/[^/]+\.py$
|
|
exclude: ^.*/.venv/.*$
|
|
require_serial: true
|
|
- id: check-cli-definition-imports
|
|
name: Check CLI definition files only import allowed modules
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_cli_definition_imports.py
|
|
language: python
|
|
files: ^.*/cli/definition\.py$
|
|
require_serial: false
|
|
- id: check-imports-in-providers
|
|
name: Check imports in providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_imports_in_providers.py
|
|
language: python
|
|
files: ^.*/src/airflow/providers/.*version_compat.*\.py$
|
|
require_serial: true
|
|
- id: check-conf-import-in-providers
|
|
name: Check conf is imported from compat SDK in providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_conf_import_in_providers.py
|
|
language: python
|
|
types: [python]
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
exclude: ^.*/common/compat/sdk\.py$
|
|
require_serial: false
|
|
- id: provider-version-compat
|
|
name: Check for correct version_compat imports in providers
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
entry: ../scripts/ci/prek/check_provider_version_compat.py
|
|
language: python
|
|
types: [python]
|
|
files: ^.*/src/airflow/providers/.*\.py$
|
|
require_serial: true
|
|
- repo: https://github.com/boidolr/ast-grep-pre-commit
|
|
rev: 13126a89e67880b0dac82cf40350abd2cfedd458 # frozen: 0.51.0
|
|
hooks:
|
|
- id: ast-grep
|
|
name: Check providers import httpx2 rather than httpx
|
|
env:
|
|
PYTHONPYCACHEPREFIX: .build/pycache
|
|
types: [python]
|
|
# Pinned to the providers config rather than relying on ast-grep walking up
|
|
# from the working directory, so a future root sgconfig.yml cannot shadow it.
|
|
args: ["--config", "./.ast-grep/sgconfig.yml"]
|