commit c4d25da3fa514ada2125b795dc95181064d69f30 Author: EvanLi <1939065721@qq.com> Date: Tue Sep 15 11:41:02 2020 +0800 feat: use Github OAuth2 token with Authorization header diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..d78b2d267 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Evan Li + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/source/.gitignore b/source/.gitignore new file mode 100644 index 000000000..909d692e5 --- /dev/null +++ b/source/.gitignore @@ -0,0 +1,3 @@ +access_token.txt +.ipynb_checkpoints +__pycache__ \ No newline at end of file diff --git a/source/common.py b/source/common.py new file mode 100644 index 000000000..781dbad36 --- /dev/null +++ b/source/common.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +import json +import requests +import time + +def get_access_token(): + with open('access_token.txt', 'r') as f: + access_token = f.read().strip() + return access_token + +def get_api_repos(API_URL): + ''' + get repos of api, return repos list + ''' + access_token = get_access_token() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Authorization': 'token {}'.format(access_token), + } + s = requests.session() + s.keep_alive = False # don't keep the session + time.sleep(3) # not get so fast + # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, + r = requests.get(API_URL,headers=headers) + if r.status_code != 200: + raise ValueError('Can not retrieve from {}'.format(api)) + repos_dict = json.loads(r.content) + repos = repos_dict['items'] + return repos \ No newline at end of file diff --git a/source/contents.md b/source/contents.md new file mode 100644 index 000000000..579410c05 --- /dev/null +++ b/source/contents.md @@ -0,0 +1,32 @@ +[Github Ranking](./README.md) +========== +**A list of the most github stars and forks repositories.** + +## Table of Contents +* [Most Stars](#most-stars) +* [Most Forks](#most-forks) +* [ActionScript](#actionscript) +* [C](#c) +* [C\#](#c-1) +* [C\+\+](#c-2) +* [Clojure](#clojure) +* [CoffeeScript](#coffeescript) +* [CSS](#css) +* [Go](#go) +* [Haskell](#haskell) +* [HTML](#html) +* [Java](#java) +* [JavaScript](#javascript) +* [Lua](#lua) +* [MATLAB](#matlab) +* [Objective\-C](#objective-c) +* [Perl](#perl) +* [PHP](#php) +* [Python](#python) +* [R](#r) +* [Ruby](#ruby) +* [Scala](#scala) +* [Shell](#shell) +* [Swift](#swift) +* [TeX](#tex) +* [Vim script](#vim-script) diff --git a/source/process.py b/source/process.py new file mode 100644 index 000000000..89840bba0 --- /dev/null +++ b/source/process.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import os +import pandas as pd +from common import get_api_repos +from writefile import write_text, write_head_contents, write_ranking_repo + +class Processor(object): + + def __init__(self): + # self.languages = ['Python'] # For test + # self.languages_md = ['Python'] # For test + self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] + self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] + # search limit 10 times per minute, with token is 30 per minute + # check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100' + self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100' + self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100' + + self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + + def get_all_repos(self): + # get all repos of most stars and forks, and different languages + + print("Get repos of most stars...") + repos_stars = get_api_repos(self.api_repo_stars) + + print("Get repos of most forks...") + repos_forks = get_api_repos(self.api_repo_forks) + + repos_languages = {} + for lang in self.languages: + print("Get most stars repos of {}...".format(lang)) + repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang)) + return repos_stars, repos_forks, repos_languages + + def write_head_contents(self): + write_head_contents('../README.md') + + def write_readme_lang_md(self): + # Most stars save + write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') + write_ranking_repo('../README.md','a',self.repos_stars[0:10]) + print("Save most stars in README.md!") + os.makedirs('../Top100',exist_ok=True) + write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') + write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) + print("Save most stars in Top100/Top-100-stars.md!\n") + + # Most forks save + write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") + write_ranking_repo('../README.md','a',self.repos_forks[0:10]) + print("Save most forks in README.md!") + write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') + write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) + print("Save most forks in Top100/Top-100-forks.md!\n") + + # Most stars in language save + for i in range(len(self.languages)): + lang = self.languages[i] + write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) + write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) + print("Save most stars of {} in README.md!".format(lang)) + write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) + write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) + print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) + + def repo_to_df(self,repos,item): + # prepare for saving data to csv file + repos_list = [] + for idx, repo in enumerate(repos): + repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repos_list.append(repo_info) + return pd.DataFrame(repos_list,columns = self.col) + + def save_to_csv(self): + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + df_all = pd.DataFrame(columns=self.col) + df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') + df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') + df_all = df_all.append(df_repos_stars,ignore_index = True) + df_all = df_all.append(df_repos_forks,ignore_index = True) + for lang in self.repos_languages.keys(): + df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) + df_all = df_all.append(df_repos_lang,ignore_index = True) + + save_date = datetime.utcnow().strftime("%Y-%m-%d") + os.makedirs('../Data',exist_ok=True) + df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') + print('Save data to Data/github-ranking-'+save_date+'.csv') \ No newline at end of file diff --git a/source/run.py b/source/run.py new file mode 100644 index 000000000..87cebcb36 --- /dev/null +++ b/source/run.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import os +from process import Processor + +if __name__=="__main__": + t1 = datetime.now() + + ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) + os.chdir(os.path.join(ROOT_PATH,'source')) + + processor = Processor() + print("Write head and contents of README.md!") + processor.write_head_contents() + print("write to readme and languages.md") + processor.write_readme_lang_md() + print("Save data to csv file") + processor.save_to_csv() + + print("Total time: {}s".format((datetime.now()-t1).total_seconds())) \ No newline at end of file diff --git a/source/writefile.py b/source/writefile.py new file mode 100644 index 000000000..aeb50470c --- /dev/null +++ b/source/writefile.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from datetime import datetime + +def write_text(file_name,method,text): + ''' + write text to file + + method: 'a'-append, 'w'-overwrite + ''' + with open(file_name, method, encoding='utf-8') as f: + f.write(text) + +def write_head_contents(file_name): + # write the head and contents of README.md + # write_head_contents('README.md') + write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + head = "*Last Automatic Update Time: {}*\n\n".format(write_time) + with open('contents.md','r') as f: + contents = f.read() + write_text(file_name,'w',head+contents) + +def write_ranking_repo(file_name,method,repos): + # method: 'a'-append or 'w'-overwrite + table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ +| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" + with open(file_name, method, encoding='utf-8') as f: + f.write(table_head) + for idx, repo in enumerate(repos): + repo_description = repo['description'] + if repo_description is not None: + repo_description = repo_description.replace('|','\|') #in case there is '|' in description + f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format( + idx + 1, + repo['name'], + repo['html_url'], + repo['stargazers_count'], + repo['forks_count'], + repo['language'], + repo['open_issues_count'], + repo_description, + repo['pushed_at'] + ) + ) + f.write('\n') \ No newline at end of file