mirror of
https://github.com/EvanLi/Github-Ranking.git
synced 2026-11-04 04:46:38 +00:00
refactor code and use Github GraphQL
This commit is contained in:
EvanLi committed
1 parent
6dfb27f7ee
commit
f7a032664d
6 files changed
+216
-240
No files matched your search
@@ -1,3 +1,4 @@
|
||||
access_token.txt
|
||||
.ipynb_checkpoints
|
||||
__pycache__
|
||||
__pycache__
|
||||
.idea/
|
||||
+40
-11
@@ -3,15 +3,43 @@ import json
|
||||
import requests
|
||||
import time
|
||||
|
||||
|
||||
def get_access_token():
|
||||
with open('access_token.txt', 'r') as f:
|
||||
with open('../access_token.txt', 'r') as f:
|
||||
access_token = f.read().strip()
|
||||
return access_token
|
||||
|
||||
|
||||
def write_text(file_name, method, text):
|
||||
"""
|
||||
write text to file
|
||||
method: 'a'-append, 'w'-overwrite
|
||||
"""
|
||||
with open(file_name, method, encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def write_ranking_repo(file_name, method, repos):
|
||||
# method: 'a'-append or 'w'-overwrite
|
||||
table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\
|
||||
| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n"
|
||||
with open(file_name, method, encoding='utf-8') as f:
|
||||
f.write(table_head)
|
||||
for idx, repo in enumerate(repos):
|
||||
repo_description = repo['description']
|
||||
if repo_description is not None:
|
||||
repo_description = repo_description.replace('|', '\|') # in case there is '|' in description
|
||||
f.write("| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n".format(
|
||||
idx + 1, repo['name'], repo['html_url'], repo['stargazers_count'], repo['forks_count'],
|
||||
repo['language'], repo['open_issues_count'], repo_description, repo['pushed_at']
|
||||
))
|
||||
f.write('\n')
|
||||
|
||||
|
||||
def get_api_repos(API_URL):
|
||||
'''
|
||||
"""
|
||||
get repos of api, return repos list
|
||||
'''
|
||||
"""
|
||||
access_token = get_access_token()
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
|
||||
@@ -21,19 +49,20 @@ def get_api_repos(API_URL):
|
||||
}
|
||||
s = requests.session()
|
||||
s.keep_alive = False # don't keep the session
|
||||
time.sleep(3) # not get so fast
|
||||
time.sleep(3) # not get so fast
|
||||
# requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False,
|
||||
r = requests.get(API_URL,headers=headers)
|
||||
r = requests.get(API_URL, headers=headers)
|
||||
if r.status_code != 200:
|
||||
raise ValueError('Can not retrieve from {}'.format(api))
|
||||
raise ValueError('Can not retrieve from {}'.format(API_URL))
|
||||
repos_dict = json.loads(r.content)
|
||||
repos = repos_dict['items']
|
||||
return repos
|
||||
|
||||
|
||||
def get_graphql_data(GQL):
|
||||
'''
|
||||
"""
|
||||
use graphql to get data
|
||||
'''
|
||||
"""
|
||||
access_token = get_access_token()
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
|
||||
@@ -44,11 +73,11 @@ def get_graphql_data(GQL):
|
||||
s = requests.session()
|
||||
s.keep_alive = False # don't keep the session
|
||||
graphql_api = "https://api.github.com/graphql"
|
||||
time.sleep(5) # not get so fast
|
||||
time.sleep(5) # not get so fast
|
||||
|
||||
# requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False,
|
||||
r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers)
|
||||
r = requests.post(url=graphql_api, json={"query": GQL}, headers=headers)
|
||||
|
||||
if r.status_code != 200:
|
||||
raise ValueError('Can not retrieve from {}'.format(GQL))
|
||||
return r.json()
|
||||
return r.json()
|
||||
@@ -1,32 +0,0 @@
|
||||
[Github Ranking](./README.md)
|
||||
==========
|
||||
**A list of the most github stars and forks repositories.**
|
||||
|
||||
## Table of Contents
|
||||
* [Most Stars](#most-stars)
|
||||
* [Most Forks](#most-forks)
|
||||
* [ActionScript](#actionscript)
|
||||
* [C](#c)
|
||||
* [C\#](#c-1)
|
||||
* [C\+\+](#c-2)
|
||||
* [Clojure](#clojure)
|
||||
* [CoffeeScript](#coffeescript)
|
||||
* [CSS](#css)
|
||||
* [Go](#go)
|
||||
* [Haskell](#haskell)
|
||||
* [HTML](#html)
|
||||
* [Java](#java)
|
||||
* [JavaScript](#javascript)
|
||||
* [Lua](#lua)
|
||||
* [MATLAB](#matlab)
|
||||
* [Objective\-C](#objective-c)
|
||||
* [Perl](#perl)
|
||||
* [PHP](#php)
|
||||
* [Python](#python)
|
||||
* [R](#r)
|
||||
* [Ruby](#ruby)
|
||||
* [Scala](#scala)
|
||||
* [Shell](#shell)
|
||||
* [Swift](#swift)
|
||||
* [TeX](#tex)
|
||||
* [Vim script](#vim-script)
|
||||
+174
-131
@@ -2,27 +2,66 @@
|
||||
from datetime import datetime
|
||||
import os
|
||||
import pandas as pd
|
||||
from common import get_api_repos, get_graphql_data
|
||||
from writefile import write_text, write_head_contents, write_ranking_repo
|
||||
from common import get_api_repos, get_graphql_data, write_text, write_ranking_repo
|
||||
import inspect
|
||||
|
||||
class Processor(object):
|
||||
# languages = ['Python'] # For test
|
||||
# languages_md = ['Python'] # For test
|
||||
# table_of_contents = """
|
||||
# * [Python](#python)""" # For test
|
||||
languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java",
|
||||
"JavaScript", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell",
|
||||
"Swift", "TeX", "Vim-script"]
|
||||
# Escape characters in markdown like # + - etc
|
||||
languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java",
|
||||
"JavaScript", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell",
|
||||
"Swift", "TeX", "Vim script"]
|
||||
table_of_contents = """
|
||||
* [ActionScript](#actionscript)
|
||||
* [C](#c)
|
||||
* [C\#](#c-1)
|
||||
* [C\+\+](#c-2)
|
||||
* [Clojure](#clojure)
|
||||
* [CoffeeScript](#coffeescript)
|
||||
* [CSS](#css)
|
||||
* [Go](#go)
|
||||
* [Haskell](#haskell)
|
||||
* [HTML](#html)
|
||||
* [Java](#java)
|
||||
* [Kotlin](#kotlin)
|
||||
* [JavaScript](#javascript)
|
||||
* [Lua](#lua)
|
||||
* [MATLAB](#matlab)
|
||||
* [Objective\-C](#objective-c)
|
||||
* [Perl](#perl)
|
||||
* [PHP](#php)
|
||||
* [Python](#python)
|
||||
* [R](#r)
|
||||
* [Ruby](#ruby)
|
||||
* [Scala](#scala)
|
||||
* [Shell](#shell)
|
||||
* [Swift](#swift)
|
||||
* [TeX](#tex)
|
||||
* [Vim script](#vim-script)"""
|
||||
|
||||
|
||||
class ProcessorREST(object):
|
||||
"""
|
||||
Github REST API v3
|
||||
Deprecating API authentication through query parameters, should send the token in the header
|
||||
ref: https://developer.github.com/changes/2020-02-10-deprecating-auth-through-query-param/
|
||||
search limit 10 times per minute, with token is 30 per minute
|
||||
check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# self.languages = ['Python'] # For test
|
||||
# self.languages_md = ['Python'] # For test
|
||||
self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go",
|
||||
"Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl",
|
||||
"PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"]
|
||||
self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go",
|
||||
"Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl",
|
||||
"PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"]
|
||||
# search limit 10 times per minute, with token is 30 per minute
|
||||
# check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit
|
||||
|
||||
self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100'
|
||||
self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100'
|
||||
self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100'
|
||||
|
||||
self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description']
|
||||
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
|
||||
'last_commit', 'description']
|
||||
self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos()
|
||||
|
||||
def get_all_repos(self):
|
||||
@@ -35,80 +74,22 @@ class Processor(object):
|
||||
repos_forks = get_api_repos(self.api_repo_forks)
|
||||
|
||||
repos_languages = {}
|
||||
for lang in self.languages:
|
||||
for lang in languages:
|
||||
print("Get most stars repos of {}...".format(lang))
|
||||
repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang))
|
||||
repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang=lang))
|
||||
return repos_stars, repos_forks, repos_languages
|
||||
|
||||
def write_head_contents(self):
|
||||
write_head_contents('../README.md')
|
||||
|
||||
def write_readme_lang_md(self):
|
||||
# Most stars save
|
||||
write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n')
|
||||
write_ranking_repo('../README.md','a',self.repos_stars[0:10])
|
||||
print("Save most stars in README.md!")
|
||||
os.makedirs('../Top100',exist_ok=True)
|
||||
write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n')
|
||||
write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars)
|
||||
print("Save most stars in Top100/Top-100-stars.md!\n")
|
||||
|
||||
# Most forks save
|
||||
write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n")
|
||||
write_ranking_repo('../README.md','a',self.repos_forks[0:10])
|
||||
print("Save most forks in README.md!")
|
||||
write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n')
|
||||
write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks)
|
||||
print("Save most forks in Top100/Top-100-forks.md!\n")
|
||||
|
||||
# Most stars in language save
|
||||
for i in range(len(self.languages)):
|
||||
lang = self.languages[i]
|
||||
write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang))
|
||||
write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10])
|
||||
print("Save most stars of {} in README.md!".format(lang))
|
||||
write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i]))
|
||||
write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang])
|
||||
print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang))
|
||||
|
||||
def repo_to_df(self,repos,item):
|
||||
# prepare for saving data to csv file
|
||||
repos_list = []
|
||||
for idx, repo in enumerate(repos):
|
||||
repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']]
|
||||
repos_list.append(repo_info)
|
||||
return pd.DataFrame(repos_list,columns = self.col)
|
||||
|
||||
def save_to_csv(self):
|
||||
# save top100 repos info to csv file in Data/github-ranking-year-month-day.md
|
||||
df_all = pd.DataFrame(columns=self.col)
|
||||
df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars')
|
||||
df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks')
|
||||
df_all = df_all.append(df_repos_stars,ignore_index = True)
|
||||
df_all = df_all.append(df_repos_forks,ignore_index = True)
|
||||
for lang in self.repos_languages.keys():
|
||||
df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang)
|
||||
df_all = df_all.append(df_repos_lang,ignore_index = True)
|
||||
|
||||
save_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
os.makedirs('../Data',exist_ok=True)
|
||||
df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8')
|
||||
print('Save data to Data/github-ranking-'+save_date+'.csv')
|
||||
|
||||
class ProcessorGQL(object):
|
||||
"""
|
||||
Github GraphQL API v4
|
||||
ref: https://docs.github.com/en/graphql
|
||||
use graphql to get data, limit 5000 points per hour
|
||||
check rate_limit with :
|
||||
curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# self.languages = ["ActionScript","C","CSharp","CPP"] # For test
|
||||
# self.languages_md = ["ActionScript","C","C\#","CPP"] # For test
|
||||
self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go",
|
||||
"Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl",
|
||||
"PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"]
|
||||
self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go",
|
||||
"Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl",
|
||||
"PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"]
|
||||
# use graphql to get data, limit 5000 points per hour
|
||||
# check rate_limit with :
|
||||
# curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql
|
||||
self.gql_format = """query{
|
||||
search(query: "%s", type: REPOSITORY, first: 100) {
|
||||
edges {
|
||||
@@ -138,14 +119,15 @@ class ProcessorGQL(object):
|
||||
}
|
||||
}
|
||||
"""
|
||||
self.gql_stars = self.gql_format % ("stars:>1000 sort:stars")
|
||||
self.gql_forks = self.gql_format % ("forks:>1000 sort:forks")
|
||||
self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars")
|
||||
self.gql_stars = self.gql_format % "stars:>1000 sort:stars"
|
||||
self.gql_forks = self.gql_format % "forks:>1000 sort:forks"
|
||||
self.gql_stars_lang = self.gql_format % "language:%s stars:>0 sort:stars"
|
||||
|
||||
self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description']
|
||||
self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos()
|
||||
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
|
||||
'last_commit', 'description']
|
||||
|
||||
def parse_gql_result(self,result):
|
||||
@staticmethod
|
||||
def parse_gql_result(result):
|
||||
res = []
|
||||
for repo in result["data"]["search"]["edges"]:
|
||||
repo_data = repo['node']
|
||||
@@ -169,70 +151,131 @@ class ProcessorGQL(object):
|
||||
print("Get repos of most stars...")
|
||||
repos_stars_gql = get_graphql_data(self.gql_stars)
|
||||
repos_stars = self.parse_gql_result(repos_stars_gql)
|
||||
print("Get repos of most stars success!")
|
||||
|
||||
print("Get repos of most forks...")
|
||||
repos_forks_gql = get_graphql_data(self.gql_forks)
|
||||
repos_forks = self.parse_gql_result(repos_forks_gql)
|
||||
print("Get repos of most forks success!")
|
||||
|
||||
repos_languages = {}
|
||||
for lang in self.languages:
|
||||
for lang in languages:
|
||||
print("Get most stars repos of {}...".format(lang))
|
||||
repos_languages[lang] = self.parse_gql_result(
|
||||
get_graphql_data(self.gql_stars_lang % (lang))
|
||||
get_graphql_data(self.gql_stars_lang % lang)
|
||||
)
|
||||
print("Get most stars repos of {} success!".format(lang))
|
||||
return repos_stars, repos_forks, repos_languages
|
||||
|
||||
def write_head_contents(self):
|
||||
write_head_contents('../README.md')
|
||||
|
||||
class WriteFile(object):
|
||||
def __init__(self, repos_stars, repos_forks, repos_languages):
|
||||
self.repos_stars = repos_stars
|
||||
self.repos_forks = repos_forks
|
||||
self.repos_languages = repos_languages
|
||||
self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues',
|
||||
'last_commit', 'description']
|
||||
self.repo_list = []
|
||||
self.repo_list.extend([{
|
||||
"desc": "Stars",
|
||||
"desc_md": "Stars",
|
||||
"title_readme": "Most Stars",
|
||||
"title_100": "Top 100 Stars",
|
||||
"file_100": "Top-100-stars.md",
|
||||
"data": repos_stars,
|
||||
"item": "top-100-stars",
|
||||
}, {
|
||||
"desc": "Forks",
|
||||
"desc_md": "Forks",
|
||||
"title_readme": "Most Forks",
|
||||
"title_100": "Top 100 Forks",
|
||||
"file_100": "Top-100-forks.md",
|
||||
"data": repos_forks,
|
||||
"item": "top-100-forks",
|
||||
}])
|
||||
for i in range(len(languages)):
|
||||
lang = languages[i]
|
||||
lang_md = languages_md[i]
|
||||
self.repo_list.append({
|
||||
"desc": "Forks",
|
||||
"desc_md": "Forks",
|
||||
"title_readme": lang_md,
|
||||
"title_100": f"Top 100 Stars in {lang_md}",
|
||||
"file_100": f"{lang}.md",
|
||||
"data": repos_languages[lang],
|
||||
"item": lang,
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def write_head_contents():
|
||||
# write the head and contents of README.md
|
||||
write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
head_contents = inspect.cleandoc("""[Github Ranking](./README.md)
|
||||
==========
|
||||
|
||||
**A list of the most github stars and forks repositories.**
|
||||
|
||||
*Last Automatic Update Time: {write_time}*
|
||||
|
||||
## Table of Contents
|
||||
|
||||
* [Most Stars](#most-stars)
|
||||
* [Most Forks](#most-forks)""".format(write_time=write_time)) + table_of_contents
|
||||
write_text("../README.md", 'w', head_contents)
|
||||
|
||||
def write_readme_lang_md(self):
|
||||
# Most stars save
|
||||
write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n')
|
||||
write_ranking_repo('../README.md','a',self.repos_stars[0:10])
|
||||
print("Save most stars in README.md!")
|
||||
os.makedirs('../Top100',exist_ok=True)
|
||||
write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n')
|
||||
write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars)
|
||||
print("Save most stars in Top100/Top-100-stars.md!\n")
|
||||
os.makedirs('../Top100', exist_ok=True)
|
||||
for repo in self.repo_list:
|
||||
# README.md
|
||||
title_readme, title_100, file_100, data = repo["title_readme"], repo["title_100"], repo["file_100"], repo["data"]
|
||||
write_text('../README.md', 'a',
|
||||
f"\n## {title_readme}\n\nThis is top 10, for more click **[{title_100}](Top100/{file_100})**\n\n")
|
||||
write_ranking_repo('../README.md', 'a', data[:10])
|
||||
print(f"Save {title_readme} in README.md!")
|
||||
|
||||
# Most forks save
|
||||
write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n")
|
||||
write_ranking_repo('../README.md','a',self.repos_forks[0:10])
|
||||
print("Save most forks in README.md!")
|
||||
write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n')
|
||||
write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks)
|
||||
print("Save most forks in Top100/Top-100-forks.md!\n")
|
||||
# Top 100 file
|
||||
write_text(f"../Top100/{file_100}", "w",
|
||||
f"[Github Ranking](../README.md)\n==========\n\n## {title_100}\n\n")
|
||||
write_ranking_repo(f"../Top100/{file_100}", 'a', data)
|
||||
print(f"Save {title_100} in Top100/{file_100}!\n")
|
||||
|
||||
# Most stars in language save
|
||||
for i in range(len(self.languages)):
|
||||
lang = self.languages[i]
|
||||
write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang))
|
||||
write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10])
|
||||
print("Save most stars of {} in README.md!".format(lang))
|
||||
write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i]))
|
||||
write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang])
|
||||
print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang))
|
||||
|
||||
def repo_to_df(self,repos,item):
|
||||
# prepare for saving data to csv file
|
||||
def repo_to_df(self, repos, item):
|
||||
# prepare for saving data to csv file
|
||||
repos_list = []
|
||||
for idx, repo in enumerate(repos):
|
||||
repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']]
|
||||
repo_info = [idx + 1, item, repo['name'], repo['stargazers_count'], repo['forks_count'], repo['language'],
|
||||
repo['html_url'], repo['owner']['login'], repo['open_issues_count'], repo['pushed_at'],
|
||||
repo['description']]
|
||||
repos_list.append(repo_info)
|
||||
return pd.DataFrame(repos_list,columns = self.col)
|
||||
return pd.DataFrame(repos_list, columns=self.col)
|
||||
|
||||
def save_to_csv(self):
|
||||
# save top100 repos info to csv file in Data/github-ranking-year-month-day.md
|
||||
# save top100 repos info to csv file in Data/github-ranking-year-month-day.md
|
||||
df_all = pd.DataFrame(columns=self.col)
|
||||
df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars')
|
||||
df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks')
|
||||
df_all = df_all.append(df_repos_stars,ignore_index = True)
|
||||
df_all = df_all.append(df_repos_forks,ignore_index = True)
|
||||
for lang in self.repos_languages.keys():
|
||||
df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang)
|
||||
df_all = df_all.append(df_repos_lang,ignore_index = True)
|
||||
for repo in self.repo_list:
|
||||
df_repos = self.repo_to_df(repos=repo["data"], item=repo["item"])
|
||||
df_all = df_all.append(df_repos, ignore_index=True)
|
||||
|
||||
save_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
os.makedirs('../Data',exist_ok=True)
|
||||
df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8')
|
||||
print('Save data to Data/github-ranking-'+save_date+'.csv')
|
||||
os.makedirs('../Data', exist_ok=True)
|
||||
df_all.to_csv('../Data/github-ranking-' + save_date + '.csv', index=False, encoding='utf-8')
|
||||
print('Save data to Data/github-ranking-' + save_date + '.csv')
|
||||
|
||||
|
||||
def run_by_gql():
|
||||
ROOT_PATH = os.path.abspath(os.path.join(__file__, "../../"))
|
||||
os.chdir(os.path.join(ROOT_PATH, 'source'))
|
||||
|
||||
# processor = ProcessorREST() # use Github REST API v3
|
||||
processor = ProcessorGQL() # use Github GraphQL API v4
|
||||
repos_stars, repos_forks, repos_languages = processor.get_all_repos()
|
||||
wt_obj = WriteFile(repos_stars, repos_forks, repos_languages)
|
||||
wt_obj.write_head_contents()
|
||||
wt_obj.write_readme_lang_md()
|
||||
wt_obj.save_to_csv()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
t1 = datetime.now()
|
||||
run_by_gql()
|
||||
print("Total time: {}s".format((datetime.now() - t1).total_seconds()))
|
||||
@@ -1,21 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from datetime import datetime
|
||||
import os
|
||||
from process import Processor,ProcessorGQL
|
||||
|
||||
if __name__=="__main__":
|
||||
t1 = datetime.now()
|
||||
|
||||
ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../"))
|
||||
os.chdir(os.path.join(ROOT_PATH,'source'))
|
||||
|
||||
# processor = Processor() # use Github REST API v3
|
||||
processor = ProcessorGQL() # use Github GraphQL API v4
|
||||
print("Write head and contents of README.md!")
|
||||
processor.write_head_contents()
|
||||
print("write to readme and languages.md")
|
||||
processor.write_readme_lang_md()
|
||||
print("Save data to csv file")
|
||||
processor.save_to_csv()
|
||||
|
||||
print("Total time: {}s".format((datetime.now()-t1).total_seconds()))
|
||||
@@ -1,44 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from datetime import datetime
|
||||
|
||||
def write_text(file_name,method,text):
|
||||
'''
|
||||
write text to file
|
||||
|
||||
method: 'a'-append, 'w'-overwrite
|
||||
'''
|
||||
with open(file_name, method, encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
def write_head_contents(file_name):
|
||||
# write the head and contents of README.md
|
||||
# write_head_contents('README.md')
|
||||
write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
head = "*Last Automatic Update Time: {}*\n\n".format(write_time)
|
||||
with open('contents.md','r') as f:
|
||||
contents = f.read()
|
||||
write_text(file_name,'w',head+contents)
|
||||
|
||||
def write_ranking_repo(file_name,method,repos):
|
||||
# method: 'a'-append or 'w'-overwrite
|
||||
table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\
|
||||
| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n"
|
||||
with open(file_name, method, encoding='utf-8') as f:
|
||||
f.write(table_head)
|
||||
for idx, repo in enumerate(repos):
|
||||
repo_description = repo['description']
|
||||
if repo_description is not None:
|
||||
repo_description = repo_description.replace('|','\|') #in case there is '|' in description
|
||||
f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format(
|
||||
idx + 1,
|
||||
repo['name'],
|
||||
repo['html_url'],
|
||||
repo['stargazers_count'],
|
||||
repo['forks_count'],
|
||||
repo['language'],
|
||||
repo['open_issues_count'],
|
||||
repo_description,
|
||||
repo['pushed_at']
|
||||
)
|
||||
)
|
||||
f.write('\n')
|
||||
Reference in new issue
Block a user