feat: use Github OAuth2 token with Authorization header

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Evan Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

source/.gitignore

@@ -0,0 +1,3 @@
+access_token.txt
+.ipynb_checkpoints
+__pycache__

source/common.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+import json
+import requests
+import time
+
+def get_access_token():
+    with open('access_token.txt', 'r') as f:
+        access_token = f.read().strip()
+    return access_token
+
+def get_api_repos(API_URL):
+    '''
+    get repos of api, return repos list
+    '''
+    access_token = get_access_token()
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        'Accept-Language': 'zh-CN,zh;q=0.9',
+        'Authorization': 'token {}'.format(access_token),
+    }
+    s = requests.session()
+    s.keep_alive = False  # don't keep the session
+    time.sleep(3) # not get so fast
+    # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False,
+    r = requests.get(API_URL,headers=headers)
+    if r.status_code != 200:
+        raise ValueError('Can not retrieve from {}'.format(api))
+    repos_dict = json.loads(r.content)
+    repos = repos_dict['items']
+    return repos

source/contents.md

@@ -0,0 +1,32 @@
+[Github Ranking](./README.md)
+==========
+**A list of the most github stars and forks repositories.**
+
+## Table of Contents
+* [Most Stars](#most-stars)
+* [Most Forks](#most-forks)
+* [ActionScript](#actionscript)
+* [C](#c)
+* [C\#](#c-1)
+* [C\+\+](#c-2)
+* [Clojure](#clojure)
+* [CoffeeScript](#coffeescript)
+* [CSS](#css)
+* [Go](#go)
+* [Haskell](#haskell)
+* [HTML](#html)
+* [Java](#java)
+* [JavaScript](#javascript)
+* [Lua](#lua)
+* [MATLAB](#matlab)
+* [Objective\-C](#objective-c)
+* [Perl](#perl)
+* [PHP](#php)
+* [Python](#python)
+* [R](#r)
+* [Ruby](#ruby)
+* [Scala](#scala)
+* [Shell](#shell)
+* [Swift](#swift)
+* [TeX](#tex)
+* [Vim script](#vim-script)

source/process.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+import os
+import pandas as pd
+from common import get_api_repos
+from writefile import write_text, write_head_contents, write_ranking_repo
+
+class Processor(object):
+
+    def __init__(self):
+        # self.languages = ['Python'] # For test
+        # self.languages_md = ['Python'] # For test
+        self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go",
+                        "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl",
+                        "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"]
+        self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go",
+                        "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl",
+                        "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"]
+        # search limit 10 times per minute, with token is 30 per minute
+        # check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit
+        self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100'
+        self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100'
+        self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100'
+
+        self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description']
+        self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos()
+
+    def get_all_repos(self):
+        # get all repos of most stars and forks, and different languages
+
+        print("Get repos of most stars...")
+        repos_stars = get_api_repos(self.api_repo_stars)
+
+        print("Get repos of most forks...")
+        repos_forks = get_api_repos(self.api_repo_forks)
+
+        repos_languages = {}
+        for lang in self.languages:
+            print("Get most stars repos of {}...".format(lang))
+            repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang))
+        return repos_stars, repos_forks, repos_languages
+
+    def write_head_contents(self):
+        write_head_contents('../README.md')
+
+    def write_readme_lang_md(self):
+        # Most stars save
+        write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n')
+        write_ranking_repo('../README.md','a',self.repos_stars[0:10])
+        print("Save most stars in README.md!")
+        os.makedirs('../Top100',exist_ok=True)
+        write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n')
+        write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars)
+        print("Save most stars in Top100/Top-100-stars.md!\n")
+
+        # Most forks save
+        write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n")
+        write_ranking_repo('../README.md','a',self.repos_forks[0:10])
+        print("Save most forks in README.md!")
+        write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n')
+        write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks)
+        print("Save most forks in Top100/Top-100-forks.md!\n")
+
+        # Most stars in language save
+        for i in range(len(self.languages)):
+            lang = self.languages[i]
+            write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang))
+            write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10])
+            print("Save most stars of {} in README.md!".format(lang))
+            write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i]))
+            write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang])
+            print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang))
+
+    def repo_to_df(self,repos,item):
+    # prepare for saving data to csv file
+        repos_list = []
+        for idx, repo in enumerate(repos):
+            repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']]
+            repos_list.append(repo_info)
+        return pd.DataFrame(repos_list,columns = self.col)
+
+    def save_to_csv(self):
+    # save top100 repos info to csv file in Data/github-ranking-year-month-day.md
+        df_all = pd.DataFrame(columns=self.col)
+        df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars')
+        df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks')
+        df_all = df_all.append(df_repos_stars,ignore_index = True)
+        df_all = df_all.append(df_repos_forks,ignore_index = True)
+        for lang in self.repos_languages.keys():
+            df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang)
+            df_all = df_all.append(df_repos_lang,ignore_index = True)
+
+        save_date = datetime.utcnow().strftime("%Y-%m-%d")
+        os.makedirs('../Data',exist_ok=True)
+        df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8')
+        print('Save data to Data/github-ranking-'+save_date+'.csv')

source/run.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+import os
+from process import Processor
+
+if __name__=="__main__":
+    t1 = datetime.now()
+
+    ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../"))
+    os.chdir(os.path.join(ROOT_PATH,'source'))
+
+    processor = Processor()
+    print("Write head and contents of README.md!")
+    processor.write_head_contents()
+    print("write to readme and languages.md")
+    processor.write_readme_lang_md()
+    print("Save data to csv file")
+    processor.save_to_csv()
+
+    print("Total time: {}s".format((datetime.now()-t1).total_seconds()))

source/writefile.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+
+def write_text(file_name,method,text):
+    '''
+    write text to file
+
+    method: 'a'-append, 'w'-overwrite
+    '''
+    with open(file_name, method, encoding='utf-8') as f:
+        f.write(text)
+
+def write_head_contents(file_name):
+    # write the head and contents of README.md
+    # write_head_contents('README.md')
+    write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    head = "*Last Automatic Update Time: {}*\n\n".format(write_time)
+    with open('contents.md','r') as f:
+        contents = f.read()
+    write_text(file_name,'w',head+contents)
+
+def write_ranking_repo(file_name,method,repos):
+    # method: 'a'-append or 'w'-overwrite
+    table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\
+| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n"
+    with open(file_name, method, encoding='utf-8') as f:
+        f.write(table_head)
+        for idx, repo in enumerate(repos):
+            repo_description = repo['description']
+            if repo_description is not None:
+                repo_description = repo_description.replace('|','\|') #in case there is '|' in description
+            f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format(
+                    idx + 1,
+                    repo['name'],
+                    repo['html_url'],
+                    repo['stargazers_count'],
+                    repo['forks_count'],
+                    repo['language'],
+                    repo['open_issues_count'],
+                    repo_description,
+                    repo['pushed_at']
+                )
+            )
+        f.write('\n')