mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Fix all build-system/requires including transitive dependencies (#62570)
* Fix all build-system/requires including transitive dependencies Add a feature to fix all build-system/requires to make build reproducibility works. The build-system requires are managed automatically by the `upgrade-important-versions` prek hook. This should not only provide reproducibility, but also it should prevent cases where transient dependency upgrade might break building sdist (for example as it happened with virtualenv 21 breaking hatchling with https://github.com/pypa/hatch/issues/2193 * Fix mypy 'Cannot infer type of lambda' error in upgrade script Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
840bcf3024
commit
94e189618a
33 files changed
+549
-62
No files matched your search
@@ -329,16 +329,18 @@ jobs:
|
||||
--hook-stage manual upgrade-important-versions || true
|
||||
if: always()
|
||||
env:
|
||||
UPGRADE_FLIT: "false"
|
||||
UPGRADE_GITPYTHON: "false"
|
||||
UPGRADE_GOLANG: "false"
|
||||
UPGRADE_HATCH: "false"
|
||||
UPGRADE_HATCHLING: "false"
|
||||
UPGRADE_MYPY: "false"
|
||||
UPGRADE_NODE_LTS: "false"
|
||||
UPGRADE_PIP: "false"
|
||||
UPGRADE_PYTHON: "false"
|
||||
UPGRADE_GOLANG: "false"
|
||||
UPGRADE_NODE_LTS: "false"
|
||||
UPGRADE_HATCH: "false"
|
||||
UPGRADE_PYYAML: "false"
|
||||
UPGRADE_GITPYTHON: "false"
|
||||
UPGRADE_RICH: "false"
|
||||
UPGRADE_RUFF: "false"
|
||||
UPGRADE_MYPY: "false"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: "Run automated upgrade for important versions minus uv (failing if needed)"
|
||||
run: |
|
||||
|
||||
@@ -17,15 +17,20 @@
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"GitPython==3.1.45",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"gitdb==4.0.12",
|
||||
"hatchling==1.27.0",
|
||||
"packaging==25.0",
|
||||
"pathspec==0.12.1",
|
||||
"GitPython==3.1.46",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"smmap==5.0.2",
|
||||
"tomli==2.2.1; python_version < '3.11'",
|
||||
"trove-classifiers==2025.9.11.17",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -17,7 +17,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -69,7 +69,19 @@ YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
|
||||
airflowctl = "airflowctl.__main__:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.version]
|
||||
|
||||
@@ -16,7 +16,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
+11
-1
@@ -17,7 +17,17 @@
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"hatchling==1.27.0",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -16,7 +16,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling==1.27.0"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
[build-system]
|
||||
requires = ["flit_core >=3.11,<4"]
|
||||
requires = ["flit_core >=3.12.0,<4"]
|
||||
build-backend = "flit_core.buildapi"
|
||||
|
||||
[project]
|
||||
|
||||
+13
-1
@@ -17,7 +17,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -17,7 +17,17 @@
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"hatchling==1.27.0",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -17,7 +17,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -17,7 +17,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -15,7 +15,19 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
@@ -17,7 +17,17 @@
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"hatchling==1.27.0",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -21,15 +21,17 @@
|
||||
# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
||||
[build-system]
|
||||
requires = [
|
||||
"GitPython==3.1.45",
|
||||
"gitdb==4.0.12",
|
||||
"hatchling==1.27.0",
|
||||
"packaging==25.0",
|
||||
"pathspec==0.12.1",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"smmap==5.0.2",
|
||||
"tomli==2.2.1; python_version < '3.11'",
|
||||
"trove-classifiers==2025.9.11.17",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -21,15 +21,17 @@
|
||||
# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
||||
[build-system]
|
||||
requires = [
|
||||
"GitPython==3.1.45",
|
||||
"gitdb==4.0.12",
|
||||
"hatchling==1.27.0",
|
||||
"packaging==25.0",
|
||||
"pathspec==0.12.1",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"smmap==5.0.2",
|
||||
"tomli==2.2.1; python_version < '3.11'",
|
||||
"trove-classifiers==2025.9.11.17",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
+10
-8
@@ -17,15 +17,17 @@
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"GitPython==3.1.45",
|
||||
"gitdb==4.0.12",
|
||||
"hatchling==1.27.0",
|
||||
"packaging==25.0",
|
||||
"pathspec==0.12.1",
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"smmap==5.0.2",
|
||||
"tomli==2.2.1; python_version < '3.11'",
|
||||
"trove-classifiers==2025.9.11.17",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.10,<3.11"
|
||||
# dependencies = [
|
||||
# "hatchling==1.27.0",
|
||||
# "hatchling==1.29.0",
|
||||
# "rich>=13.6.0",
|
||||
# "tabulate>=0.9.0",
|
||||
# ]
|
||||
|
||||
@@ -126,7 +126,7 @@ def check_private_classifier(pyproject: dict) -> bool:
|
||||
|
||||
def check_build_system(pyproject: dict, shared_path: Path) -> bool:
|
||||
build_system = pyproject.get("build-system", {})
|
||||
if build_system == {"requires": ["hatchling"], "build-backend": "hatchling.build"}:
|
||||
if build_system["build-backend"] == "hatchling.build":
|
||||
console.print(
|
||||
f" build-system is correct for [magenta]{shared_path.name}[/magenta] [bold green]OK[/bold green]"
|
||||
)
|
||||
|
||||
@@ -423,9 +423,11 @@ if UPGRADE_ALL_BY_DEFAULT and VERBOSE:
|
||||
console.print("[bright_blue]Upgrading all important versions")
|
||||
|
||||
# Package upgrade flags
|
||||
UPGRADE_FLIT_CORE: bool = get_env_bool("UPGRADE_FLIT_CORE")
|
||||
UPGRADE_GITPYTHON: bool = get_env_bool("UPGRADE_GITPYTHON")
|
||||
UPGRADE_GOLANG: bool = get_env_bool("UPGRADE_GOLANG")
|
||||
UPGRADE_HATCH: bool = get_env_bool("UPGRADE_HATCH")
|
||||
UPGRADE_HATCHLING: bool = get_env_bool("UPGRADE_HATCHLING")
|
||||
UPGRADE_MPROCS: bool = get_env_bool("UPGRADE_MPROCS")
|
||||
UPGRADE_NODE_LTS: bool = get_env_bool("UPGRADE_NODE_LTS")
|
||||
UPGRADE_PIP: bool = get_env_bool("UPGRADE_PIP")
|
||||
@@ -497,6 +499,20 @@ def apply_pattern_replacements(
|
||||
|
||||
# Configuration for packages that follow simple version constant patterns
|
||||
SIMPLE_VERSION_PATTERNS: dict[str, list[tuple[str, str]]] = {
|
||||
"flit_core": [
|
||||
(r"(flit_core==)([0-9.abrc]+)", "flit_core=={version}"),
|
||||
(r"(flit_core >=)([0-9.abrc]+)", "flit_core >={version}"),
|
||||
(r"(flit-core>=)([0-9.abrc]+)", "flit-core>={version}"),
|
||||
(r"(flit-core==)([0-9.abrc]+)", "flit-core=={version}"),
|
||||
(r"(flit>=)([0-9.abrc]+)", "flit>={version}"),
|
||||
(r"(flit==)([0-9.abrc]+)", "flit=={version}"),
|
||||
],
|
||||
"hatchling": [
|
||||
(r"(hatchling==)([0-9.abrc]+)", "hatchling=={version}"),
|
||||
(r"(hatchling>=)([0-9.abrc]+)", "hatchling>={version}"),
|
||||
(r"(HATCHLING_VERSION = )(\"[0-9.abrc]+\")", 'HATCHLING_VERSION = "{version}"'),
|
||||
(r"(HATCHLING_VERSION=)(\"[0-9.abrc]+\")", 'HATCHLING_VERSION="{version}"'),
|
||||
],
|
||||
"hatch": [
|
||||
(r"(HATCH_VERSION = )(\"[0-9.abrc]+\")", 'HATCH_VERSION = "{version}"'),
|
||||
(r"(HATCH_VERSION=)(\"[0-9.abrc]+\")", 'HATCH_VERSION="{version}"'),
|
||||
@@ -553,7 +569,9 @@ def fetch_all_package_versions() -> dict[str, str]:
|
||||
"pip": get_latest_pypi_version("pip", UPGRADE_PIP),
|
||||
"uv": get_latest_pypi_version("uv", UPGRADE_UV),
|
||||
"prek": get_latest_pypi_version("prek", UPGRADE_PREK),
|
||||
"flit_core": get_latest_pypi_version("flit_core", UPGRADE_FLIT_CORE),
|
||||
"hatch": get_latest_pypi_version("hatch", UPGRADE_HATCH),
|
||||
"hatchling": get_latest_pypi_version("hatchling", UPGRADE_HATCHLING),
|
||||
"pyyaml": get_latest_pypi_version("PyYAML", UPGRADE_PYYAML),
|
||||
"gitpython": get_latest_pypi_version("GitPython", UPGRADE_GITPYTHON),
|
||||
"ruff": get_latest_pypi_version("ruff", UPGRADE_RUFF),
|
||||
@@ -671,6 +689,167 @@ def sync_breeze_lock_file() -> None:
|
||||
)
|
||||
|
||||
|
||||
def resolve_hatchling_build_requires(with_gitpython: bool = False) -> list[str]:
|
||||
"""
|
||||
Resolve the full transitive dependency list for hatchling (with virtualenv<21) using uv pip compile.
|
||||
|
||||
When with_gitpython is True, also includes GitPython and its transitive dependencies (gitdb, smmap).
|
||||
Returns a sorted list of pinned requirement strings, with tomli carrying its python_version marker.
|
||||
"""
|
||||
packages = ["hatchling", "virtualenv<21"]
|
||||
if with_gitpython:
|
||||
packages.append("gitpython")
|
||||
|
||||
result = subprocess.run(
|
||||
["uv", "pip", "compile", "-", "--resolution", "highest", "--python-version", "3.10"],
|
||||
input="\n".join(packages) + "\n",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Parse output: lines like "package==version" (skip comment lines and blank lines)
|
||||
requires: list[str] = []
|
||||
for _line in result.stdout.splitlines():
|
||||
line = _line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Take only the "name==version" part (strip trailing comments)
|
||||
pkg_spec = line.split()[0]
|
||||
# Normalise package name to canonical casing
|
||||
pkg_name_lower = pkg_spec.split("==")[0].lower().replace("-", "_")
|
||||
pkg_version = pkg_spec.split("==")[1] if "==" in pkg_spec else ""
|
||||
if not pkg_version:
|
||||
continue
|
||||
# Use canonical casing for known packages
|
||||
CANONICAL_NAMES = {
|
||||
"gitpython": "GitPython",
|
||||
"gitdb": "gitdb",
|
||||
"smmap": "smmap",
|
||||
"hatchling": "hatchling",
|
||||
"packaging": "packaging",
|
||||
"pathspec": "pathspec",
|
||||
"pluggy": "pluggy",
|
||||
"trove_classifiers": "trove-classifiers",
|
||||
"tomli": "tomli",
|
||||
"virtualenv": "virtualenv",
|
||||
"distlib": "distlib",
|
||||
"filelock": "filelock",
|
||||
"platformdirs": "platformdirs",
|
||||
"typing_extensions": "typing-extensions",
|
||||
}
|
||||
canonical = CANONICAL_NAMES.get(pkg_name_lower, pkg_spec.split("==")[0])
|
||||
if pkg_name_lower == "tomli":
|
||||
requires.append(f"{canonical}=={pkg_version}; python_version < '3.11'")
|
||||
elif pkg_name_lower == "typing_extensions":
|
||||
# typing_extensions is built-in from Python 3.11+
|
||||
requires.append(f"{canonical}=={pkg_version}; python_version < '3.11'")
|
||||
else:
|
||||
requires.append(f"{canonical}=={pkg_version}")
|
||||
|
||||
return sorted(requires, key=lambda r: r.split("==")[0].lower())
|
||||
|
||||
|
||||
def _build_requires_block(requires: list[str]) -> str:
|
||||
"""Render the TOML requires array content (lines between the brackets)."""
|
||||
lines = []
|
||||
for req in requires:
|
||||
lines.append(f' "{req}",')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def update_pyproject_build_requires(
|
||||
hatchling_requires: list[str],
|
||||
hatchling_with_git_requires: list[str],
|
||||
versions: dict[str, str],
|
||||
) -> bool:
|
||||
"""
|
||||
Scan all pyproject.toml files in the repo (excluding out/ directory).
|
||||
|
||||
For each file:
|
||||
- If build-system/requires contains flit_core → upgrade to latest flit_core version (if set).
|
||||
- If build-system uses hatchling → replace the requires list with the resolved transitive deps.
|
||||
airflow-core/pyproject.toml gets the gitpython-inclusive list; all others get the plain list.
|
||||
|
||||
Returns True if any file was changed.
|
||||
"""
|
||||
changed = False
|
||||
|
||||
# Pattern to match and replace the full requires = [...] block under [build-system]
|
||||
# Handles both single-line and multi-line forms.
|
||||
build_system_requires_re = re.compile(
|
||||
r"(\[build-system\]\s*\n(?:[^\[]*?\n)*?requires\s*=\s*)(\[[^\]]*\])",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
flit_version = versions.get("flit_core", "")
|
||||
|
||||
for pyproject_path in sorted(AIRFLOW_ROOT_PATH.rglob("pyproject.toml")):
|
||||
# Skip anything under the out/ directory (reproducible build snapshots)
|
||||
if "out/" in pyproject_path.as_posix().replace(str(AIRFLOW_ROOT_PATH) + "/", ""):
|
||||
continue
|
||||
|
||||
content = pyproject_path.read_text()
|
||||
|
||||
# Determine if this file uses flit_core or hatchling
|
||||
build_system_match = build_system_requires_re.search(content)
|
||||
if not build_system_match:
|
||||
continue
|
||||
|
||||
requires_block = build_system_match.group(2)
|
||||
|
||||
if "flit_core" in requires_block or "flit-core" in requires_block:
|
||||
# Upgrade flit_core version if requested
|
||||
if not (UPGRADE_FLIT_CORE and flit_version):
|
||||
continue
|
||||
new_content = apply_simple_regex_replacements(
|
||||
content, flit_version, SIMPLE_VERSION_PATTERNS["flit_core"]
|
||||
)
|
||||
if new_content != content:
|
||||
pyproject_path.write_text(new_content)
|
||||
console.print(f"[bright_blue]Updated flit_core in {pyproject_path}")
|
||||
changed = True
|
||||
|
||||
elif "hatchling" in requires_block or "hatchling" in content:
|
||||
if not (UPGRADE_HATCHLING and hatchling_requires):
|
||||
continue
|
||||
# Choose the right list
|
||||
is_airflow_core = pyproject_path == AIRFLOW_CORE_ROOT_PATH / "pyproject.toml"
|
||||
target_requires = hatchling_with_git_requires if is_airflow_core else hatchling_requires
|
||||
|
||||
new_requires_lines = _build_requires_block(target_requires)
|
||||
new_requires_array = f"[\n{new_requires_lines}\n]"
|
||||
|
||||
def _replace_requires(m: re.Match[str], _new: str = new_requires_array) -> str:
|
||||
return m.group(1) + _new
|
||||
|
||||
new_content = build_system_requires_re.sub(
|
||||
_replace_requires,
|
||||
content,
|
||||
)
|
||||
|
||||
# Also apply the simple hatchling version pattern (for files that pin hatchling== elsewhere)
|
||||
hatchling_version = next(
|
||||
(
|
||||
r.split("==")[1].split(";")[0].strip()
|
||||
for r in target_requires
|
||||
if r.lower().startswith("hatchling==")
|
||||
),
|
||||
"",
|
||||
)
|
||||
if hatchling_version:
|
||||
new_content = apply_simple_regex_replacements(
|
||||
new_content, hatchling_version, SIMPLE_VERSION_PATTERNS["hatchling"]
|
||||
)
|
||||
|
||||
if new_content != content:
|
||||
pyproject_path.write_text(new_content)
|
||||
console.print(f"[bright_blue]Updated hatchling build-system requires in {pyproject_path}")
|
||||
changed = True
|
||||
|
||||
return changed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entry point for the version upgrade script."""
|
||||
retrieve_gh_token(description="airflow-upgrade-important-versions", scopes="public_repo")
|
||||
@@ -681,6 +860,19 @@ def main() -> None:
|
||||
|
||||
changed = process_all_files(versions, latest_python_versions)
|
||||
|
||||
# Resolve hatchling transitive dependencies and update all pyproject.toml build-system sections
|
||||
if UPGRADE_HATCHLING or UPGRADE_FLIT_CORE:
|
||||
console.print("[bright_blue]Resolving hatchling transitive build dependencies via uv pip compile")
|
||||
hatchling_requires = resolve_hatchling_build_requires(with_gitpython=False)
|
||||
hatchling_with_git_requires = resolve_hatchling_build_requires(with_gitpython=True)
|
||||
if VERBOSE:
|
||||
console.print(f"[bright_blue]Hatchling requires: {hatchling_requires}")
|
||||
console.print(f"[bright_blue]Hatchling+GitPython requires: {hatchling_with_git_requires}")
|
||||
pyproject_changed = update_pyproject_build_requires(
|
||||
hatchling_requires, hatchling_with_git_requires, versions
|
||||
)
|
||||
changed = changed or pyproject_changed
|
||||
|
||||
if changed:
|
||||
sync_breeze_lock_file()
|
||||
if not os.environ.get("CI"):
|
||||
|
||||
@@ -37,7 +37,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -33,7 +33,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -34,7 +34,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -35,7 +35,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -34,7 +34,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -51,7 +51,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -38,7 +38,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -41,7 +41,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -31,7 +31,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -36,7 +36,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -34,7 +34,19 @@ dev = [
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
||||
@@ -17,7 +17,19 @@
|
||||
# under the License.
|
||||
|
||||
[build-system]
|
||||
requires = [ "hatchling==1.27.0" ]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
|
||||
+13
-1
@@ -118,7 +118,19 @@ YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
requires = [
|
||||
"distlib==0.4.0",
|
||||
"filelock==3.24.3",
|
||||
"hatchling==1.29.0",
|
||||
"packaging==26.0",
|
||||
"pathspec==1.0.4",
|
||||
"platformdirs==4.9.2",
|
||||
"pluggy==1.6.0",
|
||||
"tomli==2.4.0; python_version < '3.11'",
|
||||
"trove-classifiers==2026.1.14.14",
|
||||
"typing-extensions==4.15.0; python_version < '3.11'",
|
||||
"virtualenv==20.39.1",
|
||||
]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.version]
|
||||
|
||||
Reference in new issue
Block a user