From e34d24c3b2c06c1539a3e6fec9646b4746620724 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 27 Apr 2025 17:12:14 +0200 Subject: [PATCH] Automatically refresh generated dependencies when needed (#49848) When we removed, added or moved providers, the generated provider dependencies .json file might have been not regenerated until we run `breeze ci-image build` - that led to several cases where for example release manager saw a failure when they tried to build packages and some non-existing packages were being built. This change implements more complete solution to refresh the provider json file - it will: * be created when missing even if you run breeze autocomplete (so first time autocomplete might be slower) * for any breeze command it will recalculate hash of all the provider yaml and pyproject.toml files to see if things changed and regenerate and reload providers if they did This should handle all cases nicely and fast. --- .../airflow_breeze/commands/main_command.py | 2 + .../src/airflow_breeze/global_constants.py | 59 +++++++++++++++++-- .../src/tests_common/pytest_plugin.py | 33 ++++++++++- generated/.gitignore | 2 +- .../update_providers_dependencies.py | 3 +- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index 90f91320b60..055b8c51947 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -46,6 +46,7 @@ from airflow_breeze.commands.common_options import ( from airflow_breeze.commands.production_image_commands import prod_image from airflow_breeze.commands.testing_commands import group_for_testing from airflow_breeze.configure_rich_click import click +from airflow_breeze.global_constants import generate_provider_dependencies_if_needed from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import get_console @@ -128,6 +129,7 @@ def main(ctx: click.Context, **kwargs: dict[str, Any]): check_for_rosetta_environment() check_for_python_emulation() + generate_provider_dependencies_if_needed() if not ctx.invoked_subcommand: ctx.forward(shell, extra_args={}) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index e777870e376..7733aa2d7fb 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -20,6 +20,7 @@ Global constants that are used by all other Breeze components. from __future__ import annotations +import itertools import json import platform import subprocess @@ -27,7 +28,11 @@ from enum import Enum from airflow_breeze.utils.functools_cache import clearable_cache from airflow_breeze.utils.host_info_utils import Architecture -from airflow_breeze.utils.path_utils import AIRFLOW_CORE_SOURCES_PATH, AIRFLOW_ROOT_PATH +from airflow_breeze.utils.path_utils import ( + AIRFLOW_CORE_SOURCES_PATH, + AIRFLOW_PROVIDERS_ROOT_PATH, + AIRFLOW_ROOT_PATH, +) RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]' # we should get more sophisticated logic here in the future, but for now we just check if @@ -553,16 +558,60 @@ def get_airflow_extras(): # Initialize integrations ALL_PYPROJECT_TOML_FILES = AIRFLOW_ROOT_PATH.rglob("pyproject.toml") +ALL_PROVIDER_YAML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml") +ALL_PROVIDER_PYPROJECT_TOML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml") PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "provider_info.schema.json" AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json" +AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH = ( + AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json.sha256sum" +) + UPDATE_PROVIDER_DEPENDENCIES_SCRIPT = ( AIRFLOW_ROOT_PATH / "scripts" / "ci" / "pre_commit" / "update_providers_dependencies.py" ) -if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists(): - subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()]) -with AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.open() as f: - PROVIDER_DEPENDENCIES = json.load(f) + +def _calculate_provider_deps_hash(): + import hashlib + + hasher = hashlib.sha256() + for file in sorted(itertools.chain(ALL_PROVIDER_PYPROJECT_TOML_FILES, ALL_PROVIDER_YAML_FILES)): + hasher.update(file.read_bytes()) + return hasher.hexdigest() + + +def _run_provider_dependencies_generation(calculated_hash=None) -> dict: + if calculated_hash is None: + calculated_hash = _calculate_provider_deps_hash() + AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.write_text(calculated_hash) + # We use regular print there as rich console might not be initialized yet here + print("Regenerating provider dependencies file") + subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()]) + return json.loads(AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.read_text()) + + +if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists(): + PROVIDER_DEPENDENCIES = _run_provider_dependencies_generation() +else: + PROVIDER_DEPENDENCIES = json.loads(AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.read_text()) + + +def generate_provider_dependencies_if_needed(): + regenerate_provider_dependencies = False + if ( + not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists() + or not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.exists() + ): + regenerate_provider_dependencies = True + calculated_hash = _calculate_provider_deps_hash() + else: + calculated_hash = _calculate_provider_deps_hash() + if calculated_hash.strip() != AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.read_text().strip(): + regenerate_provider_dependencies = True + if regenerate_provider_dependencies: + global PROVIDER_DEPENDENCIES + PROVIDER_DEPENDENCIES = _run_provider_dependencies_generation(calculated_hash) + DEVEL_DEPS_PATH = AIRFLOW_ROOT_PATH / "generated" / "devel_deps.txt" diff --git a/devel-common/src/tests_common/pytest_plugin.py b/devel-common/src/tests_common/pytest_plugin.py index ffe7e23f008..50ac601cd1a 100644 --- a/devel-common/src/tests_common/pytest_plugin.py +++ b/devel-common/src/tests_common/pytest_plugin.py @@ -147,12 +147,43 @@ _airflow_sources = os.getenv("AIRFLOW_SOURCES", None) AIRFLOW_ROOT_PATH = (Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[3]).resolve() AIRFLOW_CORE_SOURCES_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "src" AIRFLOW_CORE_TESTS_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "tests" +AIRFLOW_PROVIDERS_ROOT_PATH = AIRFLOW_ROOT_PATH / "providers" AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json" +AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH = ( + AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json.sha256sum" +) UPDATE_PROVIDER_DEPENDENCIES_SCRIPT = ( AIRFLOW_ROOT_PATH / "scripts" / "ci" / "pre_commit" / "update_providers_dependencies.py" ) -if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists(): +ALL_PROVIDER_YAML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml") +ALL_PROVIDER_PYPROJECT_TOML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml") + + +# Deliberately copied from breeze - we want to keep it in sync but we do not want to import code from +# Breeze here as we want to do it quickly +def _calculate_provider_deps_hash(): + import hashlib + + hasher = hashlib.sha256() + for file in sorted(itertools.chain(ALL_PROVIDER_PYPROJECT_TOML_FILES, ALL_PROVIDER_YAML_FILES)): + hasher.update(file.read_bytes()) + return hasher.hexdigest() + + +if ( + not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists() + or not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.exists() +): subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()]) +else: + calculated_provider_deps_hash = _calculate_provider_deps_hash() + if ( + calculated_provider_deps_hash.strip() + != AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.read_text().strip() + ): + subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()]) + AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.write_text(calculated_provider_deps_hash) +# End of copied code from breeze os.environ["AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES"] = "airflow.*\nunit.*\n" os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = os.fspath(AIRFLOW_CORE_TESTS_PATH / "unit" / "plugins") diff --git a/generated/.gitignore b/generated/.gitignore index 4e1b7a0ce39..9a7ab973135 100644 --- a/generated/.gitignore +++ b/generated/.gitignore @@ -1,4 +1,4 @@ -provider_dependencies.json +provider_dependencies.json* _build _inventory_cache _doctrees diff --git a/scripts/ci/pre_commit/update_providers_dependencies.py b/scripts/ci/pre_commit/update_providers_dependencies.py index d03a04f51fe..5397692c036 100755 --- a/scripts/ci/pre_commit/update_providers_dependencies.py +++ b/scripts/ci/pre_commit/update_providers_dependencies.py @@ -183,12 +183,11 @@ if __name__ == "__main__": find_all_providers_and_provider_files() num_files = len(ALL_PROVIDER_FILES) num_providers = len(ALL_PROVIDERS) - console.print(f"Found {num_providers} providers with {num_files} Python files.") + console.print(f"Refreshed {num_providers} providers with {num_files} Python files.") for file in ALL_PROVIDER_FILES: check_if_different_provider_used(file) for provider in sorted(ALL_PROVIDERS.keys()): provider_yaml_content = ALL_PROVIDERS[provider] - console.print(f"Reading dependencies for provider: {provider}") if provider in PROVIDERS: ALL_DEPENDENCIES[provider]["deps"].extend( PYPROJECT_TOML_CONTENT[provider]["project"]["dependencies"]