mirror of
https://github.com/apache/airflow.git
synced 2026-11-04 07:12:48 +00:00
Kaxil Naik
bfa4a1bb50
Replace deprecated login_or_token parameter with Auth.Token() to eliminate deprecation warnings when running the airflow-github compare command. Updates all three Github() instantiations in compare, changelog, and needs_categorization functions to use the new auth parameter.
588 lines
20 KiB
Python
Executable File
588 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = [
|
|
# "GitPython",
|
|
# "PyGithub",
|
|
# "rich-click",
|
|
# "packaging",
|
|
# "rich",
|
|
# ]
|
|
# ///
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# This tool is based on the Spark merge_spark_pr script:
|
|
# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from typing import TYPE_CHECKING
|
|
|
|
import git
|
|
import rich_click as click
|
|
from github import Auth, Github
|
|
from packaging import version
|
|
from rich.console import Console
|
|
from rich.progress import Progress
|
|
|
|
if TYPE_CHECKING:
|
|
from github.Issue import Issue
|
|
|
|
GIT_COMMIT_FIELDS = ["id", "author_name", "author_email", "date", "subject", "body"]
|
|
GIT_LOG_FORMAT = "%x1f".join(["%h", "%an", "%ae", "%ad", "%s", "%b"]) + "%x1e"
|
|
pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")
|
|
|
|
STATUS_COLOR_MAP = {
|
|
"Closed": "yellow",
|
|
"Merged": "green",
|
|
"Open": "red",
|
|
}
|
|
|
|
DEFAULT_SECTION_NAME = "Uncategorized"
|
|
|
|
|
|
def get_commits_between(
|
|
repo,
|
|
previous_version,
|
|
target_version,
|
|
files=None,
|
|
):
|
|
log_args = [f"--format={GIT_LOG_FORMAT}", previous_version + ".." + target_version]
|
|
if files:
|
|
log_args.append("--")
|
|
log_args.append(" ".join(files))
|
|
log = repo.git.log(*log_args)
|
|
log = log.strip("\n\x1e").split("\x1e")
|
|
log = [row.strip().split("\x1f") for row in log]
|
|
return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
|
|
|
|
|
|
def style_issue_status(status):
|
|
if status in STATUS_COLOR_MAP:
|
|
return click.style(status[:6].ljust(6), STATUS_COLOR_MAP[status])
|
|
return status[:6].ljust(6)
|
|
|
|
|
|
def get_issue_type(issue):
|
|
label_prefix = "type:"
|
|
issue_type = DEFAULT_SECTION_NAME
|
|
if issue.labels:
|
|
for label in issue.labels:
|
|
if label.name.startswith(label_prefix):
|
|
return label.name.replace(label_prefix, "").strip()
|
|
if label.name == "changelog:skip":
|
|
return "(skip)"
|
|
return issue_type
|
|
|
|
|
|
def build_main_commits_cache(repo: git.Repo, issue_numbers: set[int]) -> dict[int, str]:
|
|
"""Build a cache of PR number -> main branch commit SHA with a single git log operation"""
|
|
cache = {}
|
|
|
|
if not issue_numbers:
|
|
return cache
|
|
|
|
try:
|
|
# Single git log to get all main branch commits
|
|
# Use a reasonable range to avoid scanning entire history
|
|
try:
|
|
# Try to get commits since last major version (should cover most PR ranges)
|
|
log_output = repo.git.log("origin/main", "--format=%H %s", "--since=1 year ago")
|
|
except Exception:
|
|
# Fallback to last 2000 commits if date fails
|
|
log_output = repo.git.log("origin/main", "--format=%H %s", "-2000")
|
|
|
|
# Use regex to find all PR numbers in commit messages at once
|
|
import re
|
|
pr_pattern = r"\(#(\d+)\)$" # PR number at end of commit message
|
|
|
|
for commit_line in log_output.splitlines():
|
|
if not commit_line:
|
|
continue
|
|
|
|
# Find PR number at end of commit message
|
|
match = re.search(pr_pattern, commit_line)
|
|
if match:
|
|
pr_number = int(match.group(1))
|
|
if pr_number in issue_numbers:
|
|
commit_sha = commit_line.split(" ")[0]
|
|
cache[pr_number] = commit_sha
|
|
|
|
except Exception:
|
|
# Fallback to empty cache if git operation fails
|
|
pass
|
|
|
|
return cache
|
|
|
|
|
|
def get_commit_in_main_associated_with_pr(repo: git.Repo, issue: Issue, main_commits_cache: dict[int, str]) -> str | None:
|
|
"""For a PR, find the associated merged commit & return its SHA"""
|
|
if issue.pull_request:
|
|
# Use cache (should be pre-populated)
|
|
return main_commits_cache.get(issue.number)
|
|
return None
|
|
|
|
|
|
def build_cherrypicked_cache(repo: git.Repo, issue_numbers: list[int], previous_version: str | None = None) -> dict[int, bool]:
|
|
"""Build a cache of which issues are cherry-picked by doing a single git log operation"""
|
|
cache = {num: False for num in issue_numbers}
|
|
|
|
if not issue_numbers:
|
|
return cache
|
|
|
|
# Get all commits in range and process them
|
|
log_args = ["--format=%H %s"]
|
|
if previous_version:
|
|
log_args.append(previous_version + "..")
|
|
|
|
try:
|
|
log_output = repo.git.log(*log_args)
|
|
# Use regex to find all PR numbers in the entire log output at once
|
|
import re
|
|
pr_pattern = r"\(#(\d+)\)"
|
|
|
|
for commit_line in log_output.splitlines():
|
|
# Find all PR numbers in this commit
|
|
matches = re.findall(pr_pattern, commit_line)
|
|
for match in matches:
|
|
issue_num = int(match)
|
|
if issue_num in cache:
|
|
cache[issue_num] = True
|
|
except Exception:
|
|
# Fallback to individual checks if batch fails
|
|
pass
|
|
|
|
return cache
|
|
|
|
|
|
def is_cherrypicked(repo: git.Repo, issue: Issue, previous_version: str | None = None) -> bool:
|
|
"""Check if a given issue is cherry-picked in the current branch or not"""
|
|
log_args = ["--format=%H %s", f"--grep=(#{issue.number})"]
|
|
if previous_version:
|
|
log_args.append(previous_version + "..")
|
|
log_output = repo.git.log(*log_args)
|
|
|
|
for commit_line in log_output.splitlines():
|
|
# Check if the PR number exists in the commit message (handles cherry-picked commits with multiple PR numbers)
|
|
if commit_line and f"(#{issue.number})" in commit_line:
|
|
return True
|
|
return False
|
|
|
|
|
|
def is_pr(issue: Issue) -> bool:
|
|
return "apache/airflow/pull/" in issue.html_url
|
|
|
|
|
|
def files_touched(repo, commit_id: str) -> list[str]:
|
|
real_commit = repo.commit(commit_id)
|
|
return real_commit.stats.files.keys()
|
|
|
|
|
|
def is_core_commit(files: list[str]) -> bool:
|
|
# We list out things that _aren't_ core files,
|
|
# and we want to know if the commit changes anything
|
|
# outside of these designated-non-core files
|
|
|
|
non_core_files = [
|
|
# Providers
|
|
"airflow/providers/",
|
|
"airflow/provider.yaml.schema.json",
|
|
"docs/apache-airflow-providers-",
|
|
# chart
|
|
"chart/",
|
|
"docs/helm-chart/",
|
|
"clients",
|
|
# non-released docs
|
|
"COMMITTERS.rst",
|
|
"contributing_docs/",
|
|
"INTHEWILD.md",
|
|
"INSTALL",
|
|
"README.md",
|
|
"images/",
|
|
"codecov.yml",
|
|
"kubernetes-tests/",
|
|
".github/",
|
|
".pre-commit-config.yaml",
|
|
"yamllint-config.yml",
|
|
".markdownlint.yml",
|
|
"tests/",
|
|
"dev/",
|
|
"Dockerfile",
|
|
"Dockerfile.ci",
|
|
".hadolint.yaml",
|
|
"scripts/",
|
|
"docs/build_docs.py",
|
|
"docs/start_doc_server.sh",
|
|
".devcontainer/",
|
|
"docker-context-files/",
|
|
# Misc
|
|
".dockerignore",
|
|
"docs/spelling_wordlist.txt",
|
|
"docs/integration-logos/*",
|
|
"docs/exts/",
|
|
"docs/docker-stack",
|
|
"docs/README.rst",
|
|
"docker-tests/",
|
|
"kubernetes-tests/",
|
|
"helm-tests/",
|
|
".asf.yaml",
|
|
".mailmap",
|
|
"breeze-legacy",
|
|
"breeze-complete",
|
|
".rat-excludes",
|
|
".gitattributes",
|
|
".gitpod.yml",
|
|
"generated/",
|
|
]
|
|
|
|
for file in files:
|
|
for ignore in non_core_files:
|
|
if file.startswith(ignore):
|
|
break
|
|
# Handle renaming. Form: {old_name => new_name}somename.py
|
|
if file.startswith("{"):
|
|
new_files = file[1:].split(" => ")
|
|
if any(n.strip().startswith(ignore) for n in new_files):
|
|
break
|
|
else:
|
|
return True
|
|
return False
|
|
|
|
|
|
def print_changelog(sections):
|
|
for section, lines in sections.items():
|
|
if section == "(skip)":
|
|
continue
|
|
print(section)
|
|
print('"' * len(section))
|
|
for line in lines:
|
|
print("-", line)
|
|
print()
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
r"""
|
|
This tool should be used by Airflow Release Manager to verify what GitHub issues
|
|
were merged in the current working branch.
|
|
|
|
airflow-github compare <target_version> <github_token>
|
|
"""
|
|
|
|
|
|
@cli.command(short_help="Compare a GitHub target version against git merges")
|
|
@click.argument("target_version")
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
@click.option(
|
|
"--previous-version",
|
|
"previous_version",
|
|
help="Specify the previous tag on the working branch to limit"
|
|
" searching for few commits to find the cherry-picked commits",
|
|
)
|
|
@click.option("--unmerged", "show_uncherrypicked_only", help="Show unmerged PRs only", is_flag=True)
|
|
|
|
@click.option("--show-commits", help="Show commit SHAs (default: on, off when --unmerged)", is_flag=True, default=None)
|
|
|
|
def compare(target_version, github_token, previous_version=None, show_uncherrypicked_only=False, show_commits=None):
|
|
# Set smart defaults
|
|
if show_commits is None:
|
|
show_commits = not show_uncherrypicked_only # Default off for --unmerged
|
|
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
|
|
github_handler = Github(auth=Auth.Token(github_token))
|
|
|
|
# Fetch PRs and Issues separately, with merged PRs identified upfront
|
|
merged_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:merged'
|
|
)
|
|
)
|
|
closed_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:closed -is:merged'
|
|
)
|
|
)
|
|
open_prs: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:open'
|
|
)
|
|
)
|
|
|
|
# Skip fetching issues if we only care about unmerged PRs
|
|
if show_uncherrypicked_only:
|
|
issues = []
|
|
else:
|
|
issues: list[Issue] = list(
|
|
github_handler.search_issues(
|
|
f'repo:apache/airflow milestone:"Airflow {target_version}" is:issue'
|
|
)
|
|
)
|
|
|
|
# Create a merge status lookup
|
|
pr_merge_status_cache = {}
|
|
for pr in merged_prs:
|
|
pr_merge_status_cache[pr.number] = True
|
|
for pr in closed_prs:
|
|
pr_merge_status_cache[pr.number] = False
|
|
# Open PRs are neither merged nor closed, so we don't need to cache them
|
|
|
|
milestone_issues = merged_prs + closed_prs + open_prs + issues
|
|
|
|
num_cherrypicked = 0
|
|
num_uncherrypicked = Counter()
|
|
|
|
# :<18 says left align, pad to 18, :>6 says right align, pad to 6
|
|
# :<50.50 truncates after 50 chars
|
|
# !s forces as string
|
|
if show_commits:
|
|
formatstr = (
|
|
"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "
|
|
"| {title:<83.83} | {merged:<6} | {commit:>7.7} | {url}"
|
|
)
|
|
header_fields = {
|
|
"number": "NUMBER",
|
|
"typ": "TYPE",
|
|
"changelog": "CHANGELOG",
|
|
"status": "STATUS".ljust(6),
|
|
"title": "TITLE",
|
|
"merged": "CHERRY",
|
|
"commit": "COMMIT",
|
|
"url": "URL",
|
|
}
|
|
else:
|
|
formatstr = (
|
|
"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "
|
|
"| {title:<95.95} | {merged:<6} | {url}"
|
|
)
|
|
header_fields = {
|
|
"number": "NUMBER",
|
|
"typ": "TYPE",
|
|
"changelog": "CHANGELOG",
|
|
"status": "STATUS".ljust(6),
|
|
"title": "TITLE",
|
|
"merged": "CHERRY",
|
|
"commit": "", # Not used
|
|
"url": "URL",
|
|
}
|
|
|
|
print(formatstr.format(**header_fields))
|
|
milestone_issues = sorted(
|
|
milestone_issues, key=lambda x: x.closed_at if x.closed_at else x.created_at, reverse=True
|
|
)
|
|
|
|
# Build caches for performance optimization
|
|
issue_numbers = [issue.number for issue in milestone_issues if is_pr(issue)]
|
|
|
|
# Convert to set for O(1) lookups in cache building
|
|
issue_numbers_set = set(issue_numbers)
|
|
|
|
# Build all caches upfront with batch operations
|
|
if show_commits:
|
|
main_commits_cache = build_main_commits_cache(repo, issue_numbers_set)
|
|
else:
|
|
main_commits_cache = {}
|
|
|
|
cherrypicked_cache = build_cherrypicked_cache(repo, issue_numbers, previous_version)
|
|
|
|
for issue in milestone_issues:
|
|
issue_is_pr = is_pr(issue)
|
|
|
|
# Determine status - differentiate between Closed and Merged for PRs using cache
|
|
if issue_is_pr and issue.state == "closed":
|
|
is_merged = pr_merge_status_cache.get(issue.number, False)
|
|
status = "Merged" if is_merged else "Closed"
|
|
else:
|
|
status = issue.state.capitalize()
|
|
|
|
# Checks if commit was cherrypicked into branch using cache
|
|
if issue_is_pr:
|
|
is_cherry_picked = cherrypicked_cache.get(issue.number, False)
|
|
else:
|
|
is_cherry_picked = is_cherrypicked(repo, issue, previous_version)
|
|
|
|
if is_cherry_picked:
|
|
num_cherrypicked += 1
|
|
if show_uncherrypicked_only:
|
|
continue
|
|
cherrypicked = click.style("Yes".ljust(6), "green")
|
|
elif not issue_is_pr and show_uncherrypicked_only:
|
|
# Don't show issues when looking for unmerged PRs
|
|
continue
|
|
elif issue_is_pr:
|
|
num_uncherrypicked[status] += 1
|
|
cherrypicked = click.style("No".ljust(6), "red")
|
|
else:
|
|
cherrypicked = ""
|
|
|
|
fields = dict(
|
|
number=issue.number,
|
|
typ="PR" if issue_is_pr else "Issue",
|
|
changelog=get_issue_type(issue) if issue_is_pr else "",
|
|
status=style_issue_status(status),
|
|
title=issue.title,
|
|
url=issue.html_url,
|
|
)
|
|
|
|
# Only get commit info if we're showing commits
|
|
if show_commits:
|
|
commit_in_main = get_commit_in_main_associated_with_pr(repo, issue, main_commits_cache)
|
|
fields["commit"] = commit_in_main if commit_in_main else ""
|
|
|
|
print(formatstr.format(**fields, merged=cherrypicked))
|
|
|
|
print(
|
|
f"Commits on branch: {num_cherrypicked:d}, {sum(num_uncherrypicked.values()):d} "
|
|
f"({dict(num_uncherrypicked)}) yet to be cherry-picked"
|
|
)
|
|
|
|
|
|
@cli.command(short_help="Build a CHANGELOG grouped by GitHub Issue type")
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
@click.option("--disable-progress-bar", is_flag=True, help="Disable the progress bar")
|
|
def changelog(previous_version, target_version, github_token, disable_progress_bar):
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
# Get a list of issues/PRs that have been committed on the current branch.
|
|
log = get_commits_between(repo, previous_version, target_version)
|
|
|
|
if disable_progress_bar:
|
|
print(f"Processing {len(log)} commits")
|
|
|
|
gh = Github(auth=Auth.Token(github_token))
|
|
gh_repo = gh.get_repo("apache/airflow")
|
|
sections = defaultdict(list)
|
|
with open("RELEASE_NOTES.rst") as file:
|
|
existing_pr_nos = re.findall(r"\(#(\d+)\)", file.read())
|
|
|
|
console = Console(width=180)
|
|
|
|
with Progress(console=console, disable=disable_progress_bar) as progress:
|
|
task = progress.add_task("Processing commits from changelog", total=len(log))
|
|
for commit in progress.track(log, description="Processing commits from changelog"):
|
|
tickets = pr_title_re.findall(commit["subject"])
|
|
if tickets:
|
|
issue = gh_repo.get_issue(number=int(tickets[0][1:]))
|
|
issue_type = get_issue_type(issue)
|
|
files = files_touched(repo, commit["id"])
|
|
if str(issue.number) in existing_pr_nos:
|
|
print(f"Skipping {issue.number} as it's already in RELEASE_NOTES.rst")
|
|
continue
|
|
if is_core_commit(files):
|
|
sections[issue_type].append(commit["subject"])
|
|
progress.update(task, advance=1)
|
|
else:
|
|
sections[DEFAULT_SECTION_NAME].append(commit["subject"])
|
|
progress.update(task, advance=1)
|
|
|
|
console.print("\n")
|
|
print_changelog(sections)
|
|
|
|
|
|
@cli.command(short_help="Find merged PRs that still need to be categorized for the changelog")
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
@click.option("--show-skipped", is_flag=True)
|
|
@click.option("--show-files", is_flag=True)
|
|
@click.argument("github-token", envvar="GITHUB_TOKEN")
|
|
def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token):
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
log = get_commits_between(repo, previous_version, target_version)
|
|
|
|
gh = Github(auth=Auth.Token(github_token))
|
|
gh_repo = gh.get_repo("apache/airflow")
|
|
for commit in log:
|
|
tickets = pr_title_re.findall(commit["subject"])
|
|
if tickets:
|
|
issue = gh_repo.get_issue(number=int(tickets[0][1:]))
|
|
issue_type = get_issue_type(issue)
|
|
if issue_type == DEFAULT_SECTION_NAME:
|
|
files = files_touched(repo, commit["id"])
|
|
if is_core_commit(files):
|
|
print(f"{commit['subject']}: {issue.html_url}")
|
|
if show_files:
|
|
for f in files:
|
|
print(f"\t{f}")
|
|
elif show_skipped:
|
|
print(f"**** {commit['subject']}: Skipping - not a core commit")
|
|
if show_files:
|
|
for f in files:
|
|
print(f"\t{f}")
|
|
else:
|
|
print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}")
|
|
|
|
|
|
@cli.command(
|
|
name="api-clients-policy",
|
|
help="Compare two airflow core release tags and determine if API clients need to be released.",
|
|
)
|
|
@click.argument("previous_version")
|
|
@click.argument("target_version")
|
|
def api_clients_policy(previous_version, target_version):
|
|
p_version = version.parse(previous_version)
|
|
t_version = version.parse(target_version)
|
|
|
|
if p_version.major != t_version.major:
|
|
print("This is a major release, API clients should also be released.")
|
|
return
|
|
if p_version.minor != t_version.minor:
|
|
print("This is a minor release, API clients should also be released.")
|
|
return
|
|
if p_version == t_version:
|
|
print("Both versions are identical")
|
|
return
|
|
|
|
repo = git.Repo(".", search_parent_directories=True)
|
|
log = get_commits_between(
|
|
repo,
|
|
previous_version,
|
|
target_version,
|
|
files=[f"{repo.working_dir}/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml"],
|
|
)
|
|
|
|
clients_need_release = False
|
|
for commit in log:
|
|
if "update airflow version to" not in commit["subject"].lower():
|
|
clients_need_release = True
|
|
print(f"Commit '{commit['id']}' updated the OpenAPI spec, PR number: {commit['subject']}")
|
|
|
|
if clients_need_release:
|
|
print(f"API clients need to be released because the API spec has changed since '{previous_version}'")
|
|
else:
|
|
print(
|
|
"API clients don't need to be released because the API spec hasn't changed between those two "
|
|
"patch versions"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import doctest
|
|
|
|
(failure_count, test_count) = doctest.testmod()
|
|
if failure_count:
|
|
sys.exit(-1)
|
|
try:
|
|
cli()
|
|
except Exception:
|
|
raise
|