nprasad2077/Github-Ranking
1
0 Fork 0
mirror of https://github.com/EvanLi/Github-Ranking.git synced 2026-11-04 15:00:06 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
dev
Github-Ranking/source/process.py
T
EvanLi aad5986364 add languages
2021-08-19 19:58:22 +08:00

290 lines
11 KiB
Python
Raw Permalink Blame History

# -*- coding: utf-8 -*-
from datetime import datetime
import os
import pandas as pd
from common import get_api_repos, get_graphql_data, write_text, write_ranking_repo
import inspect
# languages = ['Python'] # For test
# languages_md = ['Python'] # For test
# table_of_contents = """
# * [Python](#python)""" # For test
languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Dart", "DM", "Elixir", "Go", "Groovy", "Haskell", "HTML", "Java",
"JavaScript", "Julia", "Kotlin", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "PowerShell", "Python", "R", "Ruby", "Rust", "Scala", "Shell",
"Swift", "TeX", "TypeScript", "Vim-script"]
# Escape characters in markdown like # + - etc
languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Dart", "DM", "Elixir", "Go", "Groovy", "Haskell", "HTML", "Java",
"JavaScript", "Julia", "Kotlin", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "PowerShell", "Python", "R", "Ruby", "Rust", "Scala", "Shell",
"Swift", "TeX", "TypeScript", "Vim script"]
table_of_contents = """
* [ActionScript](#actionscript)
* [C](#c)
* [C\#](#c-1)
* [C\+\+](#c-2)
* [Clojure](#clojure)
* [CoffeeScript](#coffeescript)
* [CSS](#css)
* [Dart](#dart)
* [DM](#dm)
* [Elixir](#elixir)
* [Go](#go)
* [Groovy](#groovy)
* [Haskell](#haskell)
* [HTML](#html)
* [Java](#java)
* [JavaScript](#javascript)
* [Julia](#julia)
* [Kotlin](#kotlin)
* [Lua](#lua)
* [MATLAB](#matlab)
* [Objective\-C](#objective-c)
* [Perl](#perl)
* [PHP](#php)
* [PowerShell](#powershell)
* [Python](#python)
* [R](#r)
* [Ruby](#ruby)
* [Rust](#rust)
* [Scala](#scala)
* [Shell](#shell)
* [Swift](#swift)
* [TeX](#tex)
* [TypeScript](#typeScript)
* [Vim script](#vim-script)"""
class ProcessorREST(object):
"""
Github REST API v3
Deprecating API authentication through query parameters, should send the token in the header
ref: https://developer.github.com/changes/2020-02-10-deprecating-auth-through-query-param/
search limit 10 times per minute, with token is 30 per minute
check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit
"""
def __init__(self):
self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100'
self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100'
self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100'
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
'last_commit', 'description']
self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos()
def get_all_repos(self):
# get all repos of most stars and forks, and different languages
print("Get repos of most stars...")
repos_stars = get_api_repos(self.api_repo_stars)
print("Get repos of most forks...")
repos_forks = get_api_repos(self.api_repo_forks)
repos_languages = {}
for lang in languages:
print("Get most stars repos of {}...".format(lang))
repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang=lang))
return repos_stars, repos_forks, repos_languages
class ProcessorGQL(object):
"""
Github GraphQL API v4
ref: https://docs.github.com/en/graphql
use graphql to get data, limit 5000 points per hour
check rate_limit with :
curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql
"""
def __init__(self):
self.gql_format = """query{
search(query: "%s", type: REPOSITORY, first: 100) {
edges {
node {
...on Repository {
id
name
url
forkCount
stargazers {
totalCount
}
owner {
login
}
issues(states: OPEN) {
totalCount
}
description
pushedAt
primaryLanguage {
name
}
}
}
}
}
}
"""
self.gql_stars = self.gql_format % "stars:>1000 sort:stars"
self.gql_forks = self.gql_format % "forks:>1000 sort:forks"
self.gql_stars_lang = self.gql_format % "language:%s stars:>0 sort:stars"
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
'last_commit', 'description']
@staticmethod
def parse_gql_result(result):
res = []
for repo in result["data"]["search"]["edges"]:
repo_data = repo['node']
res.append({
'name': repo_data['name'],
'stargazers_count': repo_data['stargazers']['totalCount'],
'forks_count': repo_data['forkCount'],
'language': repo_data['primaryLanguage']['name'] if repo_data['primaryLanguage'] is not None else None,
'html_url': repo_data['url'],
'owner': {
'login': repo_data['owner']['login'],
},
'open_issues_count': repo_data['issues']['totalCount'],
'pushed_at': repo_data['pushedAt'],
'description': repo_data['description'],
})
return res
def get_all_repos(self):
# get all repos of most stars and forks, and different languages
print("Get repos of most stars...")
repos_stars_gql = get_graphql_data(self.gql_stars)
repos_stars = self.parse_gql_result(repos_stars_gql)
print("Get repos of most stars success!")
print("Get repos of most forks...")
repos_forks_gql = get_graphql_data(self.gql_forks)
repos_forks = self.parse_gql_result(repos_forks_gql)
print("Get repos of most forks success!")
repos_languages = {}
for lang in languages:
print("Get most stars repos of {}...".format(lang))
repos_languages[lang] = self.parse_gql_result(
get_graphql_data(self.gql_stars_lang % lang)
)
print("Get most stars repos of {} success!".format(lang))
return repos_stars, repos_forks, repos_languages
class WriteFile(object):
def __init__(self, repos_stars, repos_forks, repos_languages):
self.repos_stars = repos_stars
self.repos_forks = repos_forks
self.repos_languages = repos_languages
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
'last_commit', 'description']
self.repo_list = []
self.repo_list.extend([{
"desc": "Stars",
"desc_md": "Stars",
"title_readme": "Most Stars",
"title_100": "Top 100 Stars",
"file_100": "Top-100-stars.md",
"data": repos_stars,
"item": "top-100-stars",
}, {
"desc": "Forks",
"desc_md": "Forks",
"title_readme": "Most Forks",
"title_100": "Top 100 Forks",
"file_100": "Top-100-forks.md",
"data": repos_forks,
"item": "top-100-forks",
}])
for i in range(len(languages)):
lang = languages[i]
lang_md = languages_md[i]
self.repo_list.append({
"desc": "Forks",
"desc_md": "Forks",
"title_readme": lang_md,
"title_100": f"Top 100 Stars in {lang_md}",
"file_100": f"{lang}.md",
"data": repos_languages[lang],
"item": lang,
})
@staticmethod
def write_head_contents():
# write the head and contents of README.md
write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
head_contents = inspect.cleandoc("""[Github Ranking](./README.md)
==========
**A list of the most github stars and forks repositories.**
*Last Automatic Update Time: {write_time}*
## Table of Contents
* [Most Stars](#most-stars)
* [Most Forks](#most-forks)""".format(write_time=write_time)) + table_of_contents
write_text("../README.md", 'w', head_contents)
def write_readme_lang_md(self):
os.makedirs('../Top100', exist_ok=True)
for repo in self.repo_list:
# README.md
title_readme, title_100, file_100, data = repo["title_readme"], repo["title_100"], repo["file_100"], repo["data"]
write_text('../README.md', 'a',
f"\n## {title_readme}\n\nThis is top 10, for more click **[{title_100}](Top100/{file_100})**\n\n")
write_ranking_repo('../README.md', 'a', data[:10])
print(f"Save {title_readme} in README.md!")
# Top 100 file
write_text(f"../Top100/{file_100}", "w",
f"[Github Ranking](../README.md)\n==========\n\n## {title_100}\n\n")
write_ranking_repo(f"../Top100/{file_100}", 'a', data)
print(f"Save {title_100} in Top100/{file_100}!\n")
def repo_to_df(self, repos, item):
# prepare for saving data to csv file
repos_list = []
for idx, repo in enumerate(repos):
repo_info = [idx + 1, item, repo['name'], repo['stargazers_count'], repo['forks_count'], repo['language'],
repo['html_url'], repo['owner']['login'], repo['open_issues_count'], repo['pushed_at'],
repo['description']]
repos_list.append(repo_info)
return pd.DataFrame(repos_list, columns=self.col)
def save_to_csv(self):
# save top100 repos info to csv file in Data/github-ranking-year-month-day.md
df_all = pd.DataFrame(columns=self.col)
for repo in self.repo_list:
df_repos = self.repo_to_df(repos=repo["data"], item=repo["item"])
df_all = df_all.append(df_repos, ignore_index=True)
save_date = datetime.utcnow().strftime("%Y-%m-%d")
os.makedirs('../Data', exist_ok=True)
df_all.to_csv('../Data/github-ranking-' + save_date + '.csv', index=False, encoding='utf-8')
print('Save data to Data/github-ranking-' + save_date + '.csv')
def run_by_gql():
ROOT_PATH = os.path.abspath(os.path.join(__file__, "../../"))
os.chdir(os.path.join(ROOT_PATH, 'source'))
# processor = ProcessorREST() # use Github REST API v3
processor = ProcessorGQL() # use Github GraphQL API v4
repos_stars, repos_forks, repos_languages = processor.get_all_repos()
wt_obj = WriteFile(repos_stars, repos_forks, repos_languages)
wt_obj.write_head_contents()
wt_obj.write_readme_lang_md()
wt_obj.save_to_csv()
if __name__ == "__main__":
t1 = datetime.now()
run_by_gql()
print("Total time: {}s".format((datetime.now() - t1).total_seconds()))
Reference in new issue View Git Blame Copy Permalink