diff --git a/source/common.py b/source/common.py index 781dbad36..c7457fec5 100644 --- a/source/common.py +++ b/source/common.py @@ -28,4 +28,27 @@ def get_api_repos(API_URL): raise ValueError('Can not retrieve from {}'.format(api)) repos_dict = json.loads(r.content) repos = repos_dict['items'] - return repos \ No newline at end of file + return repos + +def get_graphql_data(GQL): + ''' + use graphql to get data + ''' + access_token = get_access_token() + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'Accept-Language': 'zh-CN,zh;q=0.9', + 'Authorization': 'bearer {}'.format(access_token), + } + s = requests.session() + s.keep_alive = False # don't keep the session + graphql_api = "https://api.github.com/graphql" + time.sleep(5) # not get so fast + + # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, + r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers) + + if r.status_code != 200: + raise ValueError('Can not retrieve from {}'.format(GQL)) + return r.json() \ No newline at end of file diff --git a/source/process.py b/source/process.py index 89840bba0..e5962b5bd 100644 --- a/source/process.py +++ b/source/process.py @@ -2,7 +2,7 @@ from datetime import datetime import os import pandas as pd -from common import get_api_repos +from common import get_api_repos, get_graphql_data from writefile import write_text, write_head_contents, write_ranking_repo class Processor(object): @@ -79,6 +79,148 @@ class Processor(object): repos_list.append(repo_info) return pd.DataFrame(repos_list,columns = self.col) + def save_to_csv(self): + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + df_all = pd.DataFrame(columns=self.col) + df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') + df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') + df_all = df_all.append(df_repos_stars,ignore_index = True) + df_all = df_all.append(df_repos_forks,ignore_index = True) + for lang in self.repos_languages.keys(): + df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) + df_all = df_all.append(df_repos_lang,ignore_index = True) + + save_date = datetime.utcnow().strftime("%Y-%m-%d") + os.makedirs('../Data',exist_ok=True) + df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') + print('Save data to Data/github-ranking-'+save_date+'.csv') + +class ProcessorGQL(object): + + def __init__(self): + # self.languages = ["ActionScript","C","CSharp","CPP"] # For test + # self.languages_md = ["ActionScript","C","C\#","CPP"] # For test + self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] + self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", + "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", + "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] + # use graphql to get data, limit 5000 points per hour + # check rate_limit with : + # curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql + self.gql_format = """query{ + search(query: "%s", type: REPOSITORY, first: 100) { + edges { + node { + ...on Repository { + id + name + url + forkCount + stargazers { + totalCount + } + owner { + login + } + issues(states: OPEN) { + totalCount + } + description + pushedAt + primaryLanguage { + name + } + } + } + } + } + } + """ + self.gql_stars = self.gql_format % ("stars:>1000 sort:stars") + self.gql_forks = self.gql_format % ("forks:>1000 sort:forks") + self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars") + + self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + + def parse_gql_result(self,result): + res = [] + for repo in result["data"]["search"]["edges"]: + repo_data = repo['node'] + res.append({ + 'name': repo_data['name'], + 'stargazers_count': repo_data['stargazers']['totalCount'], + 'forks_count': repo_data['forkCount'], + 'language': repo_data['primaryLanguage']['name'] if repo_data['primaryLanguage'] is not None else None, + 'html_url': repo_data['url'], + 'owner': { + 'login': repo_data['owner']['login'], + }, + 'open_issues_count': repo_data['issues']['totalCount'], + 'pushed_at': repo_data['pushedAt'], + 'description': repo_data['description'], + }) + return res + + def get_all_repos(self): + # get all repos of most stars and forks, and different languages + print("Get repos of most stars...") + repos_stars_gql = get_graphql_data(self.gql_stars) + repos_stars = self.parse_gql_result(repos_stars_gql) + + print("Get repos of most forks...") + repos_forks_gql = get_graphql_data(self.gql_forks) + repos_forks = self.parse_gql_result(repos_forks_gql) + + repos_languages = {} + for lang in self.languages: + print("Get most stars repos of {}...".format(lang)) + repos_languages[lang] = self.parse_gql_result( + get_graphql_data(self.gql_stars_lang % (lang)) + ) + return repos_stars, repos_forks, repos_languages + + def write_head_contents(self): + write_head_contents('../README.md') + + def write_readme_lang_md(self): + # Most stars save + write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') + write_ranking_repo('../README.md','a',self.repos_stars[0:10]) + print("Save most stars in README.md!") + os.makedirs('../Top100',exist_ok=True) + write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') + write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) + print("Save most stars in Top100/Top-100-stars.md!\n") + + # Most forks save + write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") + write_ranking_repo('../README.md','a',self.repos_forks[0:10]) + print("Save most forks in README.md!") + write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') + write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) + print("Save most forks in Top100/Top-100-forks.md!\n") + + # Most stars in language save + for i in range(len(self.languages)): + lang = self.languages[i] + write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) + write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) + print("Save most stars of {} in README.md!".format(lang)) + write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) + write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) + print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) + + def repo_to_df(self,repos,item): + # prepare for saving data to csv file + repos_list = [] + for idx, repo in enumerate(repos): + repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repos_list.append(repo_info) + return pd.DataFrame(repos_list,columns = self.col) + def save_to_csv(self): # save top100 repos info to csv file in Data/github-ranking-year-month-day.md df_all = pd.DataFrame(columns=self.col) diff --git a/source/run.py b/source/run.py index 87cebcb36..2003ba6ce 100644 --- a/source/run.py +++ b/source/run.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from datetime import datetime import os -from process import Processor +from process import Processor,ProcessorGQL if __name__=="__main__": t1 = datetime.now() @@ -9,7 +9,8 @@ if __name__=="__main__": ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) os.chdir(os.path.join(ROOT_PATH,'source')) - processor = Processor() + # processor = Processor() # use Github REST API v3 + processor = ProcessorGQL() # use Github GraphQL API v4 print("Write head and contents of README.md!") processor.write_head_contents() print("write to readme and languages.md")