airflow/dev/airflow-github

#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This tool is based on the Spark merge_spark_pr script:
# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py

import re
import sys
from collections import Counter, defaultdict
from typing import List, Optional

import git
import rich_click as click
from github import Github
from github.Issue import Issue
from github.PullRequest import PullRequest

GIT_COMMIT_FIELDS = ['id', 'author_name', 'author_email', 'date', 'subject', 'body']
GIT_LOG_FORMAT = '%x1f'.join(['%h', '%an', '%ae', '%ad', '%s', '%b']) + '%x1e'
pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")


STATUS_COLOR_MAP = {
    'Closed': 'green',
    'Open': 'red',
}

DEFAULT_SECTION_NAME = 'Uncategorized'


def get_commits_between(repo, previous_version, target_version):
    log_args = [f'--format={GIT_LOG_FORMAT}', previous_version + ".." + target_version]
    log = repo.git.log(*log_args)
    log = log.strip('\n\x1e').split("\x1e")
    log = [row.strip().split("\x1f") for row in log]
    return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]


def style_issue_status(status):
    if status in STATUS_COLOR_MAP:
        return click.style(status[:6].ljust(6), STATUS_COLOR_MAP[status])
    return status[:6].ljust(6)


def get_issue_type(issue):
    label_prefix = "type:"
    issue_type = DEFAULT_SECTION_NAME
    if issue.labels:
        for label in issue.labels:
            if label.name.startswith(label_prefix):
                return label.name.replace(label_prefix, "").strip()
            if label.name == "changelog:skip":
                return "(skip)"
    return issue_type


def get_commit_in_main_associated_with_pr(repo: git.Repo, issue: Issue) -> Optional[str]:
    """For a PR, find the associated merged commit & return its SHA"""
    if issue.pull_request:
        commit = repo.git.log(f"--grep=#{issue.number}", "origin/main", "--format=%H")
        if commit:
            return commit
        else:
            pr: PullRequest = issue.as_pull_request()
            if pr.is_merged():
                commit = pr.merge_commit_sha
                return commit
    return None


def is_cherrypicked(repo: git.Repo, issue: Issue, previous_version: Optional[str] = None) -> bool:
    """Check if a given issue is cherry-picked in the current branch or not"""
    log_args = ['--format=%H', f"--grep=#{issue.number}"]
    if previous_version:
        log_args.append(previous_version + "..")
    log = repo.git.log(*log_args)

    if log:
        return True
    return False


def is_pr(issue: Issue) -> bool:
    return "apache/airflow/pull/" in issue.html_url


def files_touched(repo, commit_id: str) -> List[str]:
    real_commit = repo.commit(commit_id)
    return real_commit.stats.files.keys()


def is_core_commit(files: List[str]) -> bool:
    # We list out things that _aren't_ core files,
    # and we want to know if the commit changes anything
    # outside of these designated-non-core files

    non_core_files = [
        # Providers
        "airflow/providers/",
        "airflow/provider.yaml.schema.json",
        "provider_packages/",
        "docs/apache-airflow-providers-",
        # chart
        "chart/",
        "docs/helm-chart/",
        # Everything except scripts/in_container
        "scripts/ci",
        "scripts/docker",
        "scripts/systemd",
        "scripts/tools",
        "clients",
        # non-released docs
        "CONTRIBUTING.rst",
        "CONTRIBUTORS_QUICK_START.rst",
        "IMAGES.rst",
        "LOCAL_VIRTUALENV.rst",
        "PULL_REQUEST_WORKFLOW.rst",
        "INTHEWILD.md",
        "README.md",
        "BREEZE.rst",
        "CI.rst",
        # dev/release/testing
        "breeze",
        "codecov.yml",
        "pytest.ini",
        "kubernetes_tests/",
        ".github/",
        ".pre-commit-config.yaml",
        "yamllint-config.yml",
        ".markdownlint.yml",
        "tests/",
        "dev/",
        "Dockerfile.ci",
        # Misc
        "airflow/ui",
        ".dockerignore",
        "docs/spelling_wordlist.txt",
        "docs/integration-logos/*",
        "docs/exts/",
    ]

    for file in files:
        for ignore in non_core_files:
            if file.startswith(ignore):
                break
        else:
            return True
    return False


def print_changelog(sections):
    for section, lines in sections.items():
        if section == "(skip)":
            continue
        print(section)
        print('"' * len(section))
        for line in lines:
            print('-', line)
        print()


@click.group()
def cli():
    r"""
    This tool should be used by Airflow Release Manager to verify what GitHub issues
     were merged in the current working branch.

        airflow-github compare <target_version> <github_token>
    """


@cli.command(short_help='Compare a GitHub target version against git merges')
@click.argument('target_version')
@click.argument('github-token', envvar='GITHUB_TOKEN')
@click.option(
    '--previous-version',
    'previous_version',
    help="Specify the previous tag on the working branch to limit"
    " searching for few commits to find the cherry-picked commits",
)
@click.option('--unmerged', 'show_uncherrypicked_only', help="Show unmerged PRs only", is_flag=True)
def compare(target_version, github_token, previous_version=None, show_uncherrypicked_only=False):
    repo = git.Repo(".", search_parent_directories=True)

    github_handler = Github(github_token)
    milestone_issues: List[Issue] = list(
        github_handler.search_issues(f"repo:apache/airflow milestone:\"Airflow {target_version}\"")
    )

    num_cherrypicked = 0
    num_uncherrypicked = Counter()

    # :<18 says left align, pad to 18, :>6 says right align, pad to 6
    # :<50.50 truncates after 50 chars
    # !s forces as string
    formatstr = "{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} | {title:<83.83} | {merged:<6} | {commit:>7.7} | {url}"

    print(
        formatstr.format(
            number="NUMBER",
            typ="TYPE",
            changelog="CHANGELOG",
            status="STATUS".ljust(6),
            title="TITLE",
            merged="MERGED",
            commit="COMMIT",
            url="URL",
        )
    )

    for issue in milestone_issues:
        commit_in_main = get_commit_in_main_associated_with_pr(repo, issue)
        status = issue.state.capitalize()
        issue_is_pr = is_pr(issue)

        # Checks if commit was cherrypicked into branch.
        if is_cherrypicked(repo, issue, previous_version):
            num_cherrypicked += 1
            if show_uncherrypicked_only:
                continue
            cherrypicked = click.style("Yes".ljust(6), "green")
        elif not issue_is_pr and show_uncherrypicked_only:
            # Don't show issues when looking for unmerged PRs
            continue
        elif issue_is_pr:
            num_uncherrypicked[status] += 1
            cherrypicked = click.style("No".ljust(6), "red")
        else:
            cherrypicked = ""

        fields = dict(
            number=issue.number,
            typ="PR" if issue_is_pr else "Issue",
            changelog=get_issue_type(issue) if issue_is_pr else "",
            status=style_issue_status(status),
            title=issue.title,
            url=issue.html_url,
        )

        print(
            formatstr.format(**fields, merged=cherrypicked, commit=commit_in_main if commit_in_main else "")
        )

    print(
        f"Commits on branch: {num_cherrypicked:d}, {sum(num_uncherrypicked.values()):d} ({dict(num_uncherrypicked)}) yet to be cherry-picked"
    )


@cli.command(short_help='Build a CHANGELOG grouped by GitHub Issue type')
@click.argument('previous_version')
@click.argument('target_version')
@click.argument('github-token', envvar='GITHUB_TOKEN')
def changelog(previous_version, target_version, github_token):
    repo = git.Repo(".", search_parent_directories=True)
    # Get a list of issues/PRs that have been committed on the current branch.
    log = get_commits_between(repo, previous_version, target_version)

    gh = Github(github_token)
    gh_repo = gh.get_repo("apache/airflow")
    sections = defaultdict(list)
    for commit in log:
        tickets = pr_title_re.findall(commit['subject'])
        if tickets:
            issue = gh_repo.get_issue(number=int(tickets[0][1:]))
            issue_type = get_issue_type(issue)
            sections[issue_type].append(commit['subject'])
        else:
            sections[DEFAULT_SECTION_NAME].append(commit['subject'])

    print_changelog(sections)


@cli.command(short_help='Find merged PRs that still need to be categorized for the changelog')
@click.argument('previous_version')
@click.argument('target_version')
@click.option('--show-skipped', is_flag=True)
@click.option('--show-files', is_flag=True)
@click.argument('github-token', envvar='GITHUB_TOKEN')
def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token):
    repo = git.Repo(".", search_parent_directories=True)
    log = get_commits_between(repo, previous_version, target_version)

    gh = Github(github_token)
    gh_repo = gh.get_repo("apache/airflow")
    for commit in log:
        tickets = pr_title_re.findall(commit['subject'])
        if tickets:
            issue = gh_repo.get_issue(number=int(tickets[0][1:]))
            issue_type = get_issue_type(issue)
            if issue_type == DEFAULT_SECTION_NAME:
                files = files_touched(repo, commit['id'])
                if is_core_commit(files):
                    print(f"{commit['subject']}: {issue.html_url}")
                    if show_files:
                        for f in files:
                            print(f"\t{f}")
                elif show_skipped:
                    print(f"**** {commit['subject']}: Skipping - not a core commit")
                    if show_files:
                        for f in files:
                            print(f"\t{f}")
            else:
                print(f"**** {commit['subject']}: Already categorized")
        else:
            print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}")


if __name__ == "__main__":
    import doctest

    (failure_count, test_count) = doctest.testmod()
    if failure_count:
        sys.exit(-1)
    try:
        cli()
    except Exception:
        raise