feat: use Github GraphQL API v4 to get data

source/common.py

@@ -28,4 +28,27 @@ def get_api_repos(API_URL):
         raise ValueError('Can not retrieve from {}'.format(api))
     repos_dict = json.loads(r.content)
     repos = repos_dict['items']
-    return repos
+    return repos
+
+def get_graphql_data(GQL):
+    '''
+    use graphql to get data
+    '''
+    access_token = get_access_token()
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        'Accept-Language': 'zh-CN,zh;q=0.9',
+        'Authorization': 'bearer {}'.format(access_token),
+    }
+    s = requests.session()
+    s.keep_alive = False  # don't keep the session
+    graphql_api = "https://api.github.com/graphql"
+    time.sleep(5) # not get so fast
+
+    # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False,
+    r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers)
+
+    if r.status_code != 200:
+        raise ValueError('Can not retrieve from {}'.format(GQL))
+    return r.json()

source/process.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 import os
 import pandas as pd
-from common import get_api_repos
+from common import get_api_repos, get_graphql_data
 from writefile import write_text, write_head_contents, write_ranking_repo
 
 class Processor(object):
@@ -79,6 +79,148 @@ class Processor(object):
             repos_list.append(repo_info)
         return pd.DataFrame(repos_list,columns = self.col)
 
+    def save_to_csv(self):
+    # save top100 repos info to csv file in Data/github-ranking-year-month-day.md
+        df_all = pd.DataFrame(columns=self.col)
+        df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars')
+        df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks')
+        df_all = df_all.append(df_repos_stars,ignore_index = True)
+        df_all = df_all.append(df_repos_forks,ignore_index = True)
+        for lang in self.repos_languages.keys():
+            df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang)
+            df_all = df_all.append(df_repos_lang,ignore_index = True)
+
+        save_date = datetime.utcnow().strftime("%Y-%m-%d")
+        os.makedirs('../Data',exist_ok=True)
+        df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8')
+        print('Save data to Data/github-ranking-'+save_date+'.csv')
+
+class ProcessorGQL(object):
+
+    def __init__(self):
+        # self.languages = ["ActionScript","C","CSharp","CPP"] # For test
+        # self.languages_md = ["ActionScript","C","C\#","CPP"] # For test
+        self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go",
+                        "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl",
+                        "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"]
+        self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go",
+                        "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl",
+                        "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"]
+        # use graphql to get data, limit 5000 points per hour
+        # check rate_limit with :
+        # curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql
+        self.gql_format = """query{
+            search(query: "%s", type: REPOSITORY, first: 100) {
+                edges {
+                    node {
+                        ...on Repository {
+                            id
+                            name
+                            url
+                            forkCount
+                            stargazers {
+                                totalCount
+                            }
+                            owner {
+                                login
+                            }
+                            issues(states: OPEN) {
+                                totalCount
+                            }
+                            description
+                            pushedAt
+                            primaryLanguage {
+                                name
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        """
+        self.gql_stars = self.gql_format % ("stars:>1000 sort:stars")
+        self.gql_forks = self.gql_format % ("forks:>1000 sort:forks")
+        self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars")
+
+        self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description']
+        self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos()
+
+    def parse_gql_result(self,result):
+        res = []
+        for repo in result["data"]["search"]["edges"]:
+            repo_data = repo['node']
+            res.append({
+                'name': repo_data['name'],
+                'stargazers_count': repo_data['stargazers']['totalCount'],
+                'forks_count': repo_data['forkCount'],
+                'language': repo_data['primaryLanguage']['name'] if repo_data['primaryLanguage'] is not None else None,
+                'html_url': repo_data['url'],
+                'owner': {
+                    'login': repo_data['owner']['login'],
+                },
+                'open_issues_count': repo_data['issues']['totalCount'],
+                'pushed_at': repo_data['pushedAt'],
+                'description': repo_data['description'],
+            })
+        return res
+
+    def get_all_repos(self):
+        # get all repos of most stars and forks, and different languages
+        print("Get repos of most stars...")
+        repos_stars_gql = get_graphql_data(self.gql_stars)
+        repos_stars = self.parse_gql_result(repos_stars_gql)
+
+        print("Get repos of most forks...")
+        repos_forks_gql = get_graphql_data(self.gql_forks)
+        repos_forks = self.parse_gql_result(repos_forks_gql)
+
+        repos_languages = {}
+        for lang in self.languages:
+            print("Get most stars repos of {}...".format(lang))
+            repos_languages[lang] = self.parse_gql_result(
+                get_graphql_data(self.gql_stars_lang % (lang))
+            )
+        return repos_stars, repos_forks, repos_languages
+
+    def write_head_contents(self):
+        write_head_contents('../README.md')
+
+    def write_readme_lang_md(self):
+        # Most stars save
+        write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n')
+        write_ranking_repo('../README.md','a',self.repos_stars[0:10])
+        print("Save most stars in README.md!")
+        os.makedirs('../Top100',exist_ok=True)
+        write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n')
+        write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars)
+        print("Save most stars in Top100/Top-100-stars.md!\n")
+
+        # Most forks save
+        write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n")
+        write_ranking_repo('../README.md','a',self.repos_forks[0:10])
+        print("Save most forks in README.md!")
+        write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n')
+        write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks)
+        print("Save most forks in Top100/Top-100-forks.md!\n")
+
+        # Most stars in language save
+        for i in range(len(self.languages)):
+            lang = self.languages[i]
+            write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang))
+            write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10])
+            print("Save most stars of {} in README.md!".format(lang))
+            write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i]))
+            write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang])
+            print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang))
+
+    def repo_to_df(self,repos,item):
+    # prepare for saving data to csv file
+        repos_list = []
+        for idx, repo in enumerate(repos):
+            repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']]
+            repos_list.append(repo_info)
+        return pd.DataFrame(repos_list,columns = self.col)
+
     def save_to_csv(self):
     # save top100 repos info to csv file in Data/github-ranking-year-month-day.md
         df_all = pd.DataFrame(columns=self.col)

source/run.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
 import os
-from process import Processor
+from process import Processor,ProcessorGQL
 
 if __name__=="__main__":
     t1 = datetime.now()
@@ -9,7 +9,8 @@ if __name__=="__main__":
     ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../"))
     os.chdir(os.path.join(ROOT_PATH,'source'))
 
-    processor = Processor()
+    # processor = Processor() # use Github REST API v3
+    processor = ProcessorGQL() # use Github GraphQL API v4
     print("Write head and contents of README.md!")
     processor.write_head_contents()
     print("write to readme and languages.md")