mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 16:01:19 +00:00
Shahar Epstein
e853537c04
* Bump inline script metadata to Python 3.11 The prek hooks and dev scripts declare the interpreter they run under in PEP 723 metadata, and prek picks the hook interpreter from it, so this has to land before any hook starts using 3.11-only APIs. With Python 3.10 gone from the project they can also drop the tomli backport from their dependency lists, because tomllib is in the standard library from 3.11. * Refresh diagram checksums for the Python 3.11 script headers
586 lines
20 KiB
Python
Executable File
586 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.11"
|
|
# dependencies = [
|
|
# "GitPython",
|
|
# "PyGithub",
|
|
# "rich-click",
|
|
# "packaging",
|
|
# "rich",
|
|
# ]
|
|
# ///
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# This tool is based on the Spark merge_spark_pr script:
|
|
# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from typing import TYPE_CHECKING
|
|
|
|
import git
|
|
import rich_click as click
|
|
from github import Auth, Github
|
|
from packaging import version
|
|
from rich.console import Console
|
|
from rich.progress import Progress
|
|
|
|
if TYPE_CHECKING:
|
|
from github.Issue import Issue
|
|
|
|
GIT_COMMIT_FIELDS = ["id", "author_name", "author_email", "date", "subject", "body"]
|
|
GIT_LOG_FORMAT = "%x1f".join(["%h", "%an", "%ae", "%ad", "%s", "%b"]) + "%x1e"
|
|
pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")
|
|
|
|
STATUS_COLOR_MAP = {
|
|
"Closed": "yellow",
|
|
"Merged": "green",
|
|
"Open": "red",
|
|
}
|
|
|
|
DEFAULT_SECTION_NAME = "Uncategorized"
|
|
|
|
|
|
def get_commits_between(
|
|
repo,
|
|
previous_version,
|
|
target_version,
|
|
files=None,
|
|
):
|
|
log_args = [f"--format={GIT_LOG_FORMAT}", previous_version + ".." + target_version]
|
|
if files:
|
|
log_args.append("--")
|
|
log_args.append(" ".join(files))
|
|
log = repo.git.log(*log_args)
|
|
log = log.strip("\n\x1e").split("\x1e")
|
|
log = [row.strip().split("\x1f") for row in log]
|
|
return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
|
|
|
|
|
|
def style_issue_status(status):
|
|
if status in STATUS_COLOR_MAP:
|
|
return click.style(status[:6].ljust(6), STATUS_COLOR_MAP[status])
|
|
return status[:6].ljust(6)
|
|
|
|
|
|
def get_issue_type(issue):
|
|
label_prefix = "type:"
|
|
issue_type = DEFAULT_SECTION_NAME
|
|
if issue.labels:
|
|
for label in issue.labels:
|
|
if label.name.startswith(label_prefix):
|
|
return label.name.replace(label_prefix, "").strip()
|
|
if label.name == "changelog:skip":
|
|
return "(skip)"
|
|
return issue_type
|
|
|
|
|
|
def build_main_commits_cache(repo: git.Repo, issue_numbers: set[int]) -> dict[int, str]:
|
|
"""Build a cache of PR number -> main branch commit SHA with a single git log operation"""
|
|
cache = {}
|
|
|
|
if not issue_numbers:
|
|
return cache
|
|
|
|
try:
|
|
# Single git log to get all main branch commits
|
|
# Use a reasonable range to avoid scanning entire history
|
|
try:
|
|
# Try to get commits since last major version (should cover most PR ranges)
|
|
log_output = repo.git.log("origin/main", "--format=%H %s", "--since=1 year ago")
|
|
except Exception:
|
|
# Fallback to last 2000 commits if date fails
|
|
log_output = repo.git.log("origin/main", "--format=%H %s", "-2000")
|
|
|
|
# Use regex to find all PR numbers in commit messages at once
|
|
import re
|
|
pr_pattern = r"\(#(\d+)\)$" # PR number at end of commit message
|
|
|
|
for commit_line in log_output.splitlines():
|
|
if not commit_line:
|
|
continue
|
|
|
|
# Find PR number at end of commit message
|
|
match = re.search(pr_pattern, commit_line)
|
|
if match:
|
|
pr_number = int(match.group(1))
|
|
if pr_number in issue_numbers:
|
|
commit_sha = commit_line.split(" ")[0]
|
|
cache[pr_number] = commit_sha
|
|
|
|
except Exception:
|
|
# Fallback to empty cache if git operation fails
|
|
pass
|
|
|
|
return cache
|
|
|
|
|
|
def get_commit_in_main_associated_with_pr(repo: git.Repo, issue: Issue, main_commits_cache: dict[int, str]) -> str | None:
|
|
"""For a PR, find the associated merged commit & return its SHA"""
|
|
if issue.pull_request:
|
|
# Use cache (should be pre-populated)
|
|
return main_commits_cache.get(issue.number)
|
|
return None
|
|
|
|
|
|
def build_cherrypicked_cache(repo: git.Repo, issue_numbers: list[int], previous_version: str | None = None) -> dict[int, bool]:
|
|
"""Build a cache of which issues are cherry-picked by doing a single git log operation"""
|
|
cache = {num: False for num in issue_numbers}
|
|
|
|
if not issue_numbers:
|
|
return cache
|
|
|
|
# Get all commits in range and process them
|
|
log_args = ["--format=%H %s"]
|
|
if previous_version:
|
|
log_args.append(previous_version + "..")
|
|
|
|
try:
|
|
log_output = repo.git.log(*log_args)
|
|
# Use regex to find all PR numbers in the entire log output at once
|
|
import re
|
|
pr_pattern = r"\(#(\d+)\)"
|
|
|
|
for commit_line in log_output.splitlines():
|
|
# Find all PR numbers in this commit
|
|
matches = re.findall(pr_pattern, commit_line)
|
|
for match in matches:
|
|
issue_num = int(match)
|
|
if issue_num in cache:
|
|
cache[issue_num] = True
|
|
except Exception:
|
|
# Fallback to individual checks if batch fails
|
|
pass
|
|
|
|
return cache
|
|
|
|
|
|
def is_cherrypicked(repo: git.Repo, issue: Issue, previous_version: str | None = None) -> bool:
|
|
"""Check if a given issue is cherry-picked in the current branch or not"""
|
|
log_args = ["--format=%H %s", f"--grep=(#{issue.number})"]
|
|
if previous_version:
|
|
log_args.append(previous_version + "..")
|
|
log_output = repo.git.log(*log_args)
|
|
|
|
for commit_line in log_output.splitlines():
|
|
# Check if the PR number exists in the commit message (handles cherry-picked commits with multiple PR numbers)
|
|
if commit_line and f"(#{issue.number})" in commit_line:
|
|
return True
|
|
return False
|
|
|
|
|
|
def is_pr(issue: Issue) -> bool:
|
|
return "apache/airflow/pull/" in issue.html_url
|
|
|
|
|
|
def files_touched(repo, commit_id: str) -> list[str]:
|
|
real_commit = repo.commit(commit_id)
|
|
return real_commit.stats.files.keys()
|
|
|
|
|
|
def is_core_commit(files: list[str]) -> bool:
|
|
# We list out things that _aren't_ core files,
|
|
# and we want to know if the commit changes anything
|
|
# outside of these designated-non-core files
|
|
|
|
non_core_files = [
|
|
# Providers
|
|
"airflow/providers/",
|
|
"airflow/provider.yaml.schema.json",
|
|
"docs/apache-airflow-providers-",
|
|
# chart
|
|
"chart/",
|
|
"docs/helm-chart/",
|
|
"clients",
|
|
# non-released docs
|
|
"COMMITTERS.rst",
|
|
"contributing_docs/",
|
|
"INTHEWILD.md",
|
|
"INSTALL",
|
|
"README.md",
|
|
"images/",
|
|
"codecov.yml",
|
|
"kubernetes-tests/",
|
|
".github/",
|
|
".pre-commit-config.yaml",
|
|
"yamllint-config.yml",
|
|
".markdownlint.yml",
|
|
"tests/",
|
|
"dev/",
|
|
"Dockerfile",
|
|
"Dockerfile.ci",
|
|
".hadolint.yaml",
|
|
"scripts/",
|
|
"docs/build_docs.py",
|
|
"docs/start_doc_server.sh",
|
|
".devcontainer/",
|
|
"docker-context-files/",
|
|
# Misc
|
|
".dockerignore",
|
|
"docs/spelling_wordlist.txt",
|
|
"docs/integration-logos/*",
|
|
"docs/exts/",
|
|
"docs/docker-stack",
|
|
"docs/README.rst",
|
|
"docker-tests/",
|
|
"kubernetes-tests/",
|
|
"helm-tests/",
|
|
".asf.yaml",
|
|
".mailmap",
|
|
"breeze-legacy",
|
|
"breeze-complete",
|
|
".rat-excludes",
|
|
".gitattributes",
|
|
".gitpod.yml",
|
|
"generated/",
|
|
]
|
|
|
|
for file in files:
|
|
for ignore in non_core_files:
|
|
if file.startswith(ignore):
|
|
break
|
|
# Handle renaming. Form: {old_name => new_name}somename.py
|
|
if file.startswith("{"):
|
|
new_files = file[1:].split(" => ")
|
|
if any(n.strip().startswith(ignore) for n in new_files):
|
|
break
|
|
else:
|
|
return True
|
|
return False
|
|
|
|
|
|
def print_changelog(sections):
|
|
for section, lines in sections.items():
|
|
if section == "(skip)":
|
|
continue
|
|
print(section)
|
|
print('"' * len(section))
|
|
for line in lines:
|
|
print("-", line)
|
|
print()
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
r"""
|
|
This tool should be used by Airflow Release Manager to verify what GitHub issues
|
|
were merged in the current working branch.
|
|
|
|
airflow-github compare <target_version> <github_token>
|
|
"""
|
|
|
|
|
|
@cli.command(short_help="Compare a GitHub target version against git merges")
|
|
@click.argument("target_version")
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
@click.option(
|
|
"--previous-version",
|
|
"previous_version",
|
|
help="Specify the previous tag on the working branch to limit"
|
|
" searching for few commits to find the cherry-picked commits",
|
|
)
|
|
@click.option("--unmerged", "show_uncherrypicked_only", help="Show unmerged PRs only", is_flag=True)
|
|
@click.option("--show-commits", help="Show commit SHAs (default: on, off when --unmerged)", is_flag=True, default=None)
|
|
def compare(target_version, github_token, previous_version=None, show_uncherrypicked_only=False, show_commits=None):
|
|
# Set smart defaults
|
|
if show_commits is None:
|
|
show_commits = not show_uncherrypicked_only # Default off for --unmerged
|
|
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
|
|
github_handler = Github(auth=Auth.Token(github_token))
|
|
|
|
# Fetch PRs and Issues separately, with merged PRs identified upfront
|
|
merged_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:merged'
|
|
)
|
|
)
|
|
closed_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:closed -is:merged'
|
|
)
|
|
)
|
|
open_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:open'
|
|
)
|
|
)
|
|
|
|
# Skip fetching issues if we only care about unmerged PRs
|
|
if show_uncherrypicked_only:
|
|
issues = []
|
|
else:
|
|
issues: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:issue'
|
|
)
|
|
)
|
|
|
|
# Create a merge status lookup
|
|
pr_merge_status_cache = {}
|
|
for pr in merged_prs:
|
|
pr_merge_status_cache[pr.number] = True
|
|
for pr in closed_prs:
|
|
pr_merge_status_cache[pr.number] = False
|
|
# Open PRs are neither merged nor closed, so we don't need to cache them
|
|
|
|
milestone_issues = merged_prs + closed_prs + open_prs + issues
|
|
|
|
num_cherrypicked = 0
|
|
num_uncherrypicked = Counter()
|
|
|
|
# :<18 says left align, pad to 18, :>6 says right align, pad to 6
|
|
# :<50.50 truncates after 50 chars
|
|
# !s forces as string
|
|
if show_commits:
|
|
formatstr = (
|
|
"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "
|
|
"| {title:<83.83} | {merged:<6} | {commit:>7.7} | {url}"
|
|
)
|
|
header_fields = {
|
|
"number": "NUMBER",
|
|
"typ": "TYPE",
|
|
"changelog": "CHANGELOG",
|
|
"status": "STATUS".ljust(6),
|
|
"title": "TITLE",
|
|
"merged": "CHERRY",
|
|
"commit": "COMMIT",
|
|
"url": "URL",
|
|
}
|
|
else:
|
|
formatstr = (
|
|
"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "
|
|
"| {title:<95.95} | {merged:<6} | {url}"
|
|
)
|
|
header_fields = {
|
|
"number": "NUMBER",
|
|
"typ": "TYPE",
|
|
"changelog": "CHANGELOG",
|
|
"status": "STATUS".ljust(6),
|
|
"title": "TITLE",
|
|
"merged": "CHERRY",
|
|
"commit": "", # Not used
|
|
"url": "URL",
|
|
}
|
|
|
|
print(formatstr.format(**header_fields))
|
|
milestone_issues = sorted(
|
|
milestone_issues, key=lambda x: x.closed_at if x.closed_at else x.created_at, reverse=True
|
|
)
|
|
|
|
# Build caches for performance optimization
|
|
issue_numbers = [issue.number for issue in milestone_issues if is_pr(issue)]
|
|
|
|
# Convert to set for O(1) lookups in cache building
|
|
issue_numbers_set = set(issue_numbers)
|
|
|
|
# Build all caches upfront with batch operations
|
|
if show_commits:
|
|
main_commits_cache = build_main_commits_cache(repo, issue_numbers_set)
|
|
else:
|
|
main_commits_cache = {}
|
|
|
|
cherrypicked_cache = build_cherrypicked_cache(repo, issue_numbers, previous_version)
|
|
|
|
for issue in milestone_issues:
|
|
issue_is_pr = is_pr(issue)
|
|
|
|
# Determine status - differentiate between Closed and Merged for PRs using cache
|
|
if issue_is_pr and issue.state == "closed":
|
|
is_merged = pr_merge_status_cache.get(issue.number, False)
|
|
status = "Merged" if is_merged else "Closed"
|
|
else:
|
|
status = issue.state.capitalize()
|
|
|
|
# Checks if commit was cherrypicked into branch using cache
|
|
if issue_is_pr:
|
|
is_cherry_picked = cherrypicked_cache.get(issue.number, False)
|
|
else:
|
|
is_cherry_picked = is_cherrypicked(repo, issue, previous_version)
|
|
|
|
if is_cherry_picked:
|
|
num_cherrypicked += 1
|
|
if show_uncherrypicked_only:
|
|
continue
|
|
cherrypicked = click.style("Yes".ljust(6), "green")
|
|
elif not issue_is_pr and show_uncherrypicked_only:
|
|
# Don't show issues when looking for unmerged PRs
|
|
continue
|
|
elif issue_is_pr:
|
|
num_uncherrypicked[status] += 1
|
|
cherrypicked = click.style("No".ljust(6), "red")
|
|
else:
|
|
cherrypicked = ""
|
|
|
|
fields = dict(
|
|
number=issue.number,
|
|
typ="PR" if issue_is_pr else "Issue",
|
|
changelog=get_issue_type(issue) if issue_is_pr else "",
|
|
status=style_issue_status(status),
|
|
title=issue.title,
|
|
url=issue.html_url,
|
|
)
|
|
|
|
# Only get commit info if we're showing commits
|
|
if show_commits:
|
|
commit_in_main = get_commit_in_main_associated_with_pr(repo, issue, main_commits_cache)
|
|
fields["commit"] = commit_in_main if commit_in_main else ""
|
|
|
|
print(formatstr.format(**fields, merged=cherrypicked))
|
|
|
|
print(
|
|
f"Commits on branch: {num_cherrypicked:d}, {sum(num_uncherrypicked.values()):d} "
|
|
f"({dict(num_uncherrypicked)}) yet to be cherry-picked"
|
|
)
|
|
|
|
|
|
@cli.command(short_help="Build a CHANGELOG grouped by GitHub Issue type")
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
@click.option("--disable-progress-bar", is_flag=True, help="Disable the progress bar")
|
|
def changelog(previous_version, target_version, github_token, disable_progress_bar):
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
# Get a list of issues/PRs that have been committed on the current branch.
|
|
log = get_commits_between(repo, previous_version, target_version)
|
|
|
|
if disable_progress_bar:
|
|
print(f"Processing {len(log)} commits")
|
|
|
|
gh = Github(auth=Auth.Token(github_token))
|
|
gh_repo = gh.get_repo("apache/airflow")
|
|
sections = defaultdict(list)
|
|
with open("RELEASE_NOTES.rst") as file:
|
|
existing_pr_nos = re.findall(r"\(#(\d+)\)", file.read())
|
|
|
|
console = Console(width=180)
|
|
|
|
with Progress(console=console, disable=disable_progress_bar) as progress:
|
|
task = progress.add_task("Processing commits from changelog", total=len(log))
|
|
for commit in progress.track(log, description="Processing commits from changelog"):
|
|
tickets = pr_title_re.findall(commit["subject"])
|
|
if tickets:
|
|
issue = gh_repo.get_issue(number=int(tickets[0][1:]))
|
|
issue_type = get_issue_type(issue)
|
|
files = files_touched(repo, commit["id"])
|
|
if str(issue.number) in existing_pr_nos:
|
|
print(f"Skipping {issue.number} as it's already in RELEASE_NOTES.rst")
|
|
continue
|
|
if is_core_commit(files):
|
|
sections[issue_type].append(commit["subject"])
|
|
progress.update(task, advance=1)
|
|
else:
|
|
sections[DEFAULT_SECTION_NAME].append(commit["subject"])
|
|
progress.update(task, advance=1)
|
|
|
|
console.print("\n")
|
|
print_changelog(sections)
|
|
|
|
|
|
@cli.command(short_help="Find merged PRs that still need to be categorized for the changelog")
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
@click.option("--show-skipped", is_flag=True)
|
|
@click.option("--show-files", is_flag=True)
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token):
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
log = get_commits_between(repo, previous_version, target_version)
|
|
|
|
gh = Github(auth=Auth.Token(github_token))
|
|
gh_repo = gh.get_repo("apache/airflow")
|
|
for commit in log:
|
|
tickets = pr_title_re.findall(commit["subject"])
|
|
if tickets:
|
|
issue = gh_repo.get_issue(number=int(tickets[0][1:]))
|
|
issue_type = get_issue_type(issue)
|
|
if issue_type == DEFAULT_SECTION_NAME:
|
|
files = files_touched(repo, commit["id"])
|
|
if is_core_commit(files):
|
|
print(f"{commit['subject']}: {issue.html_url}")
|
|
if show_files:
|
|
for f in files:
|
|
print(f"\t{f}")
|
|
elif show_skipped:
|
|
print(f"**** {commit['subject']}: Skipping - not a core commit")
|
|
if show_files:
|
|
for f in files:
|
|
print(f"\t{f}")
|
|
else:
|
|
print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}")
|
|
|
|
|
|
@cli.command(
|
|
name="api-clients-policy",
|
|
help="Compare two airflow core release tags and determine if API clients need to be released.",
|
|
)
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
def api_clients_policy(previous_version, target_version):
|
|
p_version = version.parse(previous_version)
|
|
t_version = version.parse(target_version)
|
|
|
|
if p_version.major != t_version.major:
|
|
print("This is a major release, API clients should also be released.")
|
|
return
|
|
if p_version.minor != t_version.minor:
|
|
print("This is a minor release, API clients should also be released.")
|
|
return
|
|
if p_version == t_version:
|
|
print("Both versions are identical")
|
|
return
|
|
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
log = get_commits_between(
|
|
repo,
|
|
previous_version,
|
|
target_version,
|
|
files=[f"{repo.working_dir}/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml"],
|
|
)
|
|
|
|
clients_need_release = False
|
|
for commit in log:
|
|
if "update airflow version to" not in commit["subject"].lower():
|
|
clients_need_release = True
|
|
print(f"Commit '{commit['id']}' updated the OpenAPI spec, PR number: {commit['subject']}")
|
|
|
|
if clients_need_release:
|
|
print(f"API clients need to be released because the API spec has changed since '{previous_version}'")
|
|
else:
|
|
print(
|
|
"API clients don't need to be released because the API spec hasn't changed between those two "
|
|
"patch versions"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import doctest
|
|
|
|
(failure_count, test_count) = doctest.testmod()
|
|
if failure_count:
|
|
sys.exit(-1)
|
|
try:
|
|
cli()
|
|
except Exception:
|
|
raise
|