From a9ad129058a104d9bd70bb86eca0c92f9d89d0cf Mon Sep 17 00:00:00 2001 From: "Jason(Zhe-You) Liu" <68415893+jason810496@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:09:23 +0800 Subject: [PATCH] Skip allowlisted non-PEP 440 docs versions when publishing docs to S3 (#76020) * Use an explicit allowlist for skipped docs versions * Cover an excluded version listed under another package --------- --- .../src/airflow_breeze/global_constants.py | 6 +++ .../utils/publish_docs_to_s3.py | 15 +++--- dev/breeze/tests/test_publish_docs_to_s3.py | 46 ++++++++++++++----- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index e129805779d..2b259570604 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -366,6 +366,12 @@ SCHEMA_DESTINATION_LOCATIONS = [ PACKAGES_METADATA_EXCLUDE_NAMES = ["docker-stack", "apache-airflow-providers"] +# Published docs folders that are not PEP 440 versions, such as Maven snapshots staged for testing. +# Any other invalid version folder still fails the docs publishing. +PACKAGES_METADATA_EXCLUDE_VERSIONS: dict[str, list[str]] = { + "java-sdk": ["1.0.0-SNAPSHOT"], +} + @clearable_cache def all_selective_core_test_types() -> tuple[str, ...]: diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py index 7c8fb990596..e6722be51ef 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py @@ -25,7 +25,10 @@ from functools import cached_property import boto3 -from airflow_breeze.global_constants import PACKAGES_METADATA_EXCLUDE_NAMES +from airflow_breeze.global_constants import ( + PACKAGES_METADATA_EXCLUDE_NAMES, + PACKAGES_METADATA_EXCLUDE_VERSIONS, +) from airflow_breeze.utils.console import console_print from airflow_breeze.utils.parallel import check_async_run_results, run_with_pool @@ -47,9 +50,6 @@ s3_client = boto3.client("s3") cloudfront_client = boto3.client("cloudfront") -SNAPSHOT_SUFFIX = "-SNAPSHOT" - - class VersionError: """Class to track version errors during processing.""" @@ -347,11 +347,10 @@ class S3DocsPublish: console_print(f"[info]Getting package versions for {package_name} from:\n") console_print(versions) all_versions: list[Version] = [] + excluded_versions = PACKAGES_METADATA_EXCLUDE_VERSIONS.get(package_name, []) for v in versions: - # A Maven development build such as the Java SDK's `1.0.0-SNAPSHOT` is not a release and - # is not a valid Python version either; one that reached S3 is left out, not an error. - if v.upper().endswith(SNAPSHOT_SUFFIX): - console_print(f"[warning]Skipping development build {v} of {package_name}\n") + if v in excluded_versions: + console_print(f"[info]Skipping excluded version {package_name}/{v}\n") continue try: all_versions.append(Version(v)) diff --git a/dev/breeze/tests/test_publish_docs_to_s3.py b/dev/breeze/tests/test_publish_docs_to_s3.py index 531b1ff0a2a..08fe58b7f53 100644 --- a/dev/breeze/tests/test_publish_docs_to_s3.py +++ b/dev/breeze/tests/test_publish_docs_to_s3.py @@ -299,19 +299,43 @@ class TestPublishDocsToS3: assert mock_generate_packages_metadata.call_count == expected_calls + @patch.dict( + "airflow_breeze.utils.publish_docs_to_s3.PACKAGES_METADATA_EXCLUDE_VERSIONS", + {"java-sdk": ["1.0.0-SNAPSHOT"], "snapshot-only-sdk": ["1.0.0-SNAPSHOT"]}, + clear=True, + ) + @patch.object(VersionError, "version_error", False) + def test_dump_docs_package_metadata_skips_excluded_versions(self): + metadata = self.publish_docs_to_s3.dump_docs_package_metadata( + { + "java-sdk": ["1.0.0-SNAPSHOT", "1.0.0-beta1"], + "snapshot-only-sdk": ["1.0.0-SNAPSHOT"], + } + ) -@pytest.mark.parametrize( - ("versions", "expected"), - [ - pytest.param(["1.0.0-SNAPSHOT", "1.0.0-beta1"], ["1.0.0b1"], id="maven-snapshot-skipped"), - pytest.param(["2.1.0-snapshot", "2.0.0"], ["2.0.0"], id="lowercase-snapshot-skipped"), - ], -) -def test_get_latest_minor_versions_skips_development_builds(versions, expected): - VersionError.set_version_error(False) + assert metadata == [ + {"package-name": "java-sdk", "all-versions": ["1.0.0b1"], "stable-version": "1.0.0b1"}, + ] + assert not VersionError.has_any_error() - assert S3DocsPublish.get_latest_minor_versions("java-sdk", versions) == expected - assert VersionError.has_any_error() is False + @patch.dict( + "airflow_breeze.utils.publish_docs_to_s3.PACKAGES_METADATA_EXCLUDE_VERSIONS", + {"java-sdk": ["1.0.0-SNAPSHOT"]}, + clear=True, + ) + @pytest.mark.parametrize( + ("package_name", "invalid_version"), + [ + pytest.param("java-sdk", "1.1.0-SNAPSHOT", id="version_not_excluded"), + pytest.param("ts-sdk", "1.0.0-SNAPSHOT", id="version_excluded_for_another_package"), + ], + ) + @patch.object(VersionError, "version_error", False) + def test_get_latest_minor_versions_flags_versions_not_excluded(self, package_name, invalid_version): + versions = S3DocsPublish.get_latest_minor_versions(package_name, [invalid_version, "1.0.0-beta1"]) + + assert versions == ["1.0.0b1"] + assert VersionError.has_any_error() def test_get_latest_minor_versions_still_fails_on_other_invalid_versions():