Add dev script to find core PRs missing categorization (#21432)

dev/airflow-github

@@ -53,10 +53,12 @@ STATUS_COLOR_MAP = {
 DEFAULT_SECTION_NAME = 'Uncategorized'
 
 
-def get_commits_between_tags(repo, earlier_tag, later_tag):
-    log_args = ['--format="%H"', earlier_tag + ".." + later_tag]
+def get_commits_between(repo, previous_version, target_version):
+    log_args = [f'--format={GIT_LOG_FORMAT}', previous_version + ".." + target_version]
     log = repo.git.log(*log_args)
-    return list(log.strip('"').split('"\n"'))
+    log = log.strip('\n\x1e').split("\x1e")
+    log = [row.strip().split("\x1f") for row in log]
+    return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
 
 
 def style_issue_status(status):
@@ -107,6 +109,71 @@ def is_pr(issue: Issue) -> bool:
     return "apache/airflow/pull/" in issue.html_url
 
 
+def files_touched(repo, commit_id: str) -> List[str]:
+    real_commit = repo.commit(commit_id)
+    return real_commit.stats.files.keys()
+
+
+def is_core_commit(files: List[str]) -> bool:
+    # We list out things that _aren't_ core files,
+    # and we want to know if the commit changes anything
+    # outside of these designated-non-core files
+
+    non_core_files = [
+        # Providers
+        "airflow/providers/",
+        "airflow/provider.yaml.schema.json",
+        "provider_packages/",
+        "docs/apache-airflow-providers-",
+        # chart
+        "chart/",
+        "docs/helm-chart/",
+        # Everything except scripts/in_container
+        "scripts/ci",
+        "scripts/docker",
+        "scripts/systemd",
+        "scripts/tools",
+        "clients",
+        # non-released docs
+        "CONTRIBUTING.rst",
+        "CONTRIBUTORS_QUICK_START.rst",
+        "IMAGES.rst",
+        "LOCAL_VIRTUALENV.rst",
+        "PULL_REQUEST_WORKFLOW.rst",
+        "INTHEWILD.md",
+        "README.md",
+        "BREEZE.rst",
+        "CI.rst",
+        # dev/release/testing
+        "breeze",
+        "codecov.yml",
+        "confirm",
+        "pytest.ini",
+        "kubernetes_tests/",
+        ".github/",
+        ".pre-commit-config.yaml",
+        "yamllint-config.yml",
+        ".markdownlint.yml",
+        "tests/",
+        "dev/",
+        "Dockerfile.ci",
+        # Misc
+        "airflow/ui",
+        ".dockerignore",
+        "docs/spelling_wordlist.txt",
+        "docs/integration-logos/*",
+        "docs/exts/",
+    ]
+
+    for file in files:
+        for ignore in non_core_files:
+            if file.startswith(ignore):
+                break
+        else:
+            return True
+    return False
+
+
 def print_changelog(sections):
     for section, lines in sections.items():
         if section == "(skip)":
@@ -212,12 +279,7 @@ def compare(target_version, github_token, previous_version=None, show_uncherrypi
 def changelog(previous_version, target_version, github_token):
     repo = git.Repo(".", search_parent_directories=True)
     # Get a list of issues/PRs that have been committed on the current branch.
-    log_args = [f'--format={GIT_LOG_FORMAT}', previous_version + ".." + target_version]
-    log = repo.git.log(*log_args)
-
-    log = log.strip('\n\x1e').split("\x1e")
-    log = [row.strip().split("\x1f") for row in log]
-    log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
+    log = get_commits_between(repo, previous_version, target_version)
 
     gh = Github(github_token)
     gh_repo = gh.get_repo("apache/airflow")
@@ -234,6 +296,41 @@ def changelog(previous_version, target_version, github_token):
     print_changelog(sections)
 
 
+@cli.command(short_help='Find merged PRs that still need to be categorized for the changelog')
+@click.argument('previous_version')
+@click.argument('target_version')
+@click.option('--show-skipped', is_flag=True)
+@click.option('--show-files', is_flag=True)
+@click.argument('github-token', envvar='GITHUB_TOKEN')
+def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token):
+    repo = git.Repo(".", search_parent_directories=True)
+    log = get_commits_between(repo, previous_version, target_version)
+
+    gh = Github(github_token)
+    gh_repo = gh.get_repo("apache/airflow")
+    for commit in log:
+        tickets = pr_title_re.findall(commit['subject'])
+        if tickets:
+            issue = gh_repo.get_issue(number=int(tickets[0][1:]))
+            issue_type = get_issue_type(issue)
+            if issue_type == DEFAULT_SECTION_NAME:
+                files = files_touched(repo, commit['id'])
+                if is_core_commit(files):
+                    print(f"{commit['subject']}: {issue.html_url}")
+                    if show_files:
+                        for f in files:
+                            print(f"\t{f}")
+                elif show_skipped:
+                    print(f"**** {commit['subject']}: Skipping - not a core commit")
+                    if show_files:
+                        for f in files:
+                            print(f"\t{f}")
+            else:
+                print(f"**** {commit['subject']}: Already categorized")
+        else:
+            print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}")
+
+
 if __name__ == "__main__":
     import doctest