Automatically refresh generated dependencies when needed (#49848)

When we removed, added or moved providers, the generated provider
dependencies .json file might have been not regenerated until we
run `breeze ci-image build` - that led to several cases where for
example release manager saw a failure when they tried to build
packages and some non-existing packages were being built.

This change implements more complete solution to refresh the provider
json file - it will:

* be created when missing even if you run breeze autocomplete (so first
  time autocomplete might be slower)
* for any breeze command it will recalculate hash of all the provider
  yaml and pyproject.toml files to see if things changed and regenerate
  and reload providers if they did

This should handle all cases nicely and fast.

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -46,6 +46,7 @@ from airflow_breeze.commands.common_options import (
 from airflow_breeze.commands.production_image_commands import prod_image
 from airflow_breeze.commands.testing_commands import group_for_testing
 from airflow_breeze.configure_rich_click import click
+from airflow_breeze.global_constants import generate_provider_dependencies_if_needed
 from airflow_breeze.utils.click_utils import BreezeGroup
 from airflow_breeze.utils.confirm import Answer, user_confirm
 from airflow_breeze.utils.console import get_console
@@ -128,6 +129,7 @@ def main(ctx: click.Context, **kwargs: dict[str, Any]):
 
     check_for_rosetta_environment()
     check_for_python_emulation()
+    generate_provider_dependencies_if_needed()
 
     if not ctx.invoked_subcommand:
         ctx.forward(shell, extra_args={})

dev/breeze/src/airflow_breeze/global_constants.py

@@ -20,6 +20,7 @@ Global constants that are used by all other Breeze components.
 
 from __future__ import annotations
 
+import itertools
 import json
 import platform
 import subprocess
@@ -27,7 +28,11 @@ from enum import Enum
 
 from airflow_breeze.utils.functools_cache import clearable_cache
 from airflow_breeze.utils.host_info_utils import Architecture
-from airflow_breeze.utils.path_utils import AIRFLOW_CORE_SOURCES_PATH, AIRFLOW_ROOT_PATH
+from airflow_breeze.utils.path_utils import (
+    AIRFLOW_CORE_SOURCES_PATH,
+    AIRFLOW_PROVIDERS_ROOT_PATH,
+    AIRFLOW_ROOT_PATH,
+)
 
 RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
 # we should get more sophisticated logic here in the future, but for now we just check if
@@ -553,16 +558,60 @@ def get_airflow_extras():
 
 # Initialize integrations
 ALL_PYPROJECT_TOML_FILES = AIRFLOW_ROOT_PATH.rglob("pyproject.toml")
+ALL_PROVIDER_YAML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml")
+ALL_PROVIDER_PYPROJECT_TOML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml")
 PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "provider_info.schema.json"
 AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json"
+AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH = (
+    AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json.sha256sum"
+)
+
 UPDATE_PROVIDER_DEPENDENCIES_SCRIPT = (
     AIRFLOW_ROOT_PATH / "scripts" / "ci" / "pre_commit" / "update_providers_dependencies.py"
 )
-if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists():
-    subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()])
 
-with AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.open() as f:
-    PROVIDER_DEPENDENCIES = json.load(f)
+
+def _calculate_provider_deps_hash():
+    import hashlib
+
+    hasher = hashlib.sha256()
+    for file in sorted(itertools.chain(ALL_PROVIDER_PYPROJECT_TOML_FILES, ALL_PROVIDER_YAML_FILES)):
+        hasher.update(file.read_bytes())
+    return hasher.hexdigest()
+
+
+def _run_provider_dependencies_generation(calculated_hash=None) -> dict:
+    if calculated_hash is None:
+        calculated_hash = _calculate_provider_deps_hash()
+    AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.write_text(calculated_hash)
+    # We use regular print there as rich console might not be initialized yet here
+    print("Regenerating provider dependencies file")
+    subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()])
+    return json.loads(AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.read_text())
+
+
+if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists():
+    PROVIDER_DEPENDENCIES = _run_provider_dependencies_generation()
+else:
+    PROVIDER_DEPENDENCIES = json.loads(AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.read_text())
+
+
+def generate_provider_dependencies_if_needed():
+    regenerate_provider_dependencies = False
+    if (
+        not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists()
+        or not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.exists()
+    ):
+        regenerate_provider_dependencies = True
+        calculated_hash = _calculate_provider_deps_hash()
+    else:
+        calculated_hash = _calculate_provider_deps_hash()
+        if calculated_hash.strip() != AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.read_text().strip():
+            regenerate_provider_dependencies = True
+    if regenerate_provider_dependencies:
+        global PROVIDER_DEPENDENCIES
+        PROVIDER_DEPENDENCIES = _run_provider_dependencies_generation(calculated_hash)
+
 
 DEVEL_DEPS_PATH = AIRFLOW_ROOT_PATH / "generated" / "devel_deps.txt"
 

devel-common/src/tests_common/pytest_plugin.py

@@ -147,12 +147,43 @@ _airflow_sources = os.getenv("AIRFLOW_SOURCES", None)
 AIRFLOW_ROOT_PATH = (Path(_airflow_sources) if _airflow_sources else Path(__file__).parents[3]).resolve()
 AIRFLOW_CORE_SOURCES_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "src"
 AIRFLOW_CORE_TESTS_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "tests"
+AIRFLOW_PROVIDERS_ROOT_PATH = AIRFLOW_ROOT_PATH / "providers"
 AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json"
+AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH = (
+    AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json.sha256sum"
+)
 UPDATE_PROVIDER_DEPENDENCIES_SCRIPT = (
     AIRFLOW_ROOT_PATH / "scripts" / "ci" / "pre_commit" / "update_providers_dependencies.py"
 )
-if not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists():
+ALL_PROVIDER_YAML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml")
+ALL_PROVIDER_PYPROJECT_TOML_FILES = AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml")
+
+
+# Deliberately copied from breeze - we want to keep it in sync but we do not want to import code from
+# Breeze here as we want to do it quickly
+def _calculate_provider_deps_hash():
+    import hashlib
+
+    hasher = hashlib.sha256()
+    for file in sorted(itertools.chain(ALL_PROVIDER_PYPROJECT_TOML_FILES, ALL_PROVIDER_YAML_FILES)):
+        hasher.update(file.read_bytes())
+    return hasher.hexdigest()
+
+
+if (
+    not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_PATH.exists()
+    or not AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.exists()
+):
     subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()])
+else:
+    calculated_provider_deps_hash = _calculate_provider_deps_hash()
+    if (
+        calculated_provider_deps_hash.strip()
+        != AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.read_text().strip()
+    ):
+        subprocess.check_call(["uv", "run", UPDATE_PROVIDER_DEPENDENCIES_SCRIPT.as_posix()])
+        AIRFLOW_GENERATED_PROVIDER_DEPENDENCIES_HASH_PATH.write_text(calculated_provider_deps_hash)
+# End of copied code from breeze
 
 os.environ["AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES"] = "airflow.*\nunit.*\n"
 os.environ["AIRFLOW__CORE__PLUGINS_FOLDER"] = os.fspath(AIRFLOW_CORE_TESTS_PATH / "unit" / "plugins")

generated/.gitignore

@@ -1,4 +1,4 @@
-provider_dependencies.json
+provider_dependencies.json*
 _build
 _inventory_cache
 _doctrees

scripts/ci/pre_commit/update_providers_dependencies.py

@@ -183,12 +183,11 @@ if __name__ == "__main__":
     find_all_providers_and_provider_files()
     num_files = len(ALL_PROVIDER_FILES)
     num_providers = len(ALL_PROVIDERS)
-    console.print(f"Found {num_providers} providers with {num_files} Python files.")
+    console.print(f"Refreshed {num_providers} providers with {num_files} Python files.")
     for file in ALL_PROVIDER_FILES:
         check_if_different_provider_used(file)
     for provider in sorted(ALL_PROVIDERS.keys()):
         provider_yaml_content = ALL_PROVIDERS[provider]
-        console.print(f"Reading dependencies for provider: {provider}")
         if provider in PROVIDERS:
             ALL_DEPENDENCIES[provider]["deps"].extend(
                 PYPROJECT_TOML_CONTENT[provider]["project"]["dependencies"]