From f7a032664d34ce96fab1a09546befd5680ac8869 Mon Sep 17 00:00:00 2001 From: EvanLi Date: Thu, 19 Aug 2021 18:57:31 +0800 Subject: [PATCH] refactor code and use Github GraphQL --- source/.gitignore => .gitignore | 3 +- source/common.py | 51 ++++-- source/contents.md | 32 ---- source/process.py | 305 ++++++++++++++++++-------------- source/run.py | 21 --- source/writefile.py | 44 ----- 6 files changed, 216 insertions(+), 240 deletions(-) rename source/.gitignore => .gitignore (65%) delete mode 100644 source/contents.md delete mode 100644 source/run.py delete mode 100644 source/writefile.py diff --git a/source/.gitignore b/.gitignore similarity index 65% rename from source/.gitignore rename to .gitignore index 909d692e5..4fad37e93 100644 --- a/source/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ access_token.txt .ipynb_checkpoints -__pycache__ \ No newline at end of file +__pycache__ +.idea/ diff --git a/source/common.py b/source/common.py index c7457fec5..f4c53892c 100644 --- a/source/common.py +++ b/source/common.py @@ -3,15 +3,43 @@ import json import requests import time + def get_access_token(): - with open('access_token.txt', 'r') as f: + with open('../access_token.txt', 'r') as f: access_token = f.read().strip() return access_token + +def write_text(file_name, method, text): + """ + write text to file + method: 'a'-append, 'w'-overwrite + """ + with open(file_name, method, encoding='utf-8') as f: + f.write(text) + + +def write_ranking_repo(file_name, method, repos): + # method: 'a'-append or 'w'-overwrite + table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ +| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" + with open(file_name, method, encoding='utf-8') as f: + f.write(table_head) + for idx, repo in enumerate(repos): + repo_description = repo['description'] + if repo_description is not None: + repo_description = repo_description.replace('|', '\|') # in case there is '|' in description + f.write("| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n".format( + idx + 1, repo['name'], repo['html_url'], repo['stargazers_count'], repo['forks_count'], + repo['language'], repo['open_issues_count'], repo_description, repo['pushed_at'] + )) + f.write('\n') + + def get_api_repos(API_URL): - ''' + """ get repos of api, return repos list - ''' + """ access_token = get_access_token() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', @@ -21,19 +49,20 @@ def get_api_repos(API_URL): } s = requests.session() s.keep_alive = False # don't keep the session - time.sleep(3) # not get so fast + time.sleep(3) # not get so fast # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, - r = requests.get(API_URL,headers=headers) + r = requests.get(API_URL, headers=headers) if r.status_code != 200: - raise ValueError('Can not retrieve from {}'.format(api)) + raise ValueError('Can not retrieve from {}'.format(API_URL)) repos_dict = json.loads(r.content) repos = repos_dict['items'] return repos + def get_graphql_data(GQL): - ''' + """ use graphql to get data - ''' + """ access_token = get_access_token() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36', @@ -44,11 +73,11 @@ def get_graphql_data(GQL): s = requests.session() s.keep_alive = False # don't keep the session graphql_api = "https://api.github.com/graphql" - time.sleep(5) # not get so fast + time.sleep(5) # not get so fast # requests.packages.urllib3.disable_warnings() # disable InsecureRequestWarning of verify=False, - r = requests.post(url = graphql_api, json={"query": GQL}, headers=headers) + r = requests.post(url=graphql_api, json={"query": GQL}, headers=headers) if r.status_code != 200: raise ValueError('Can not retrieve from {}'.format(GQL)) - return r.json() \ No newline at end of file + return r.json() diff --git a/source/contents.md b/source/contents.md deleted file mode 100644 index 579410c05..000000000 --- a/source/contents.md +++ /dev/null @@ -1,32 +0,0 @@ -[Github Ranking](./README.md) -========== -**A list of the most github stars and forks repositories.** - -## Table of Contents -* [Most Stars](#most-stars) -* [Most Forks](#most-forks) -* [ActionScript](#actionscript) -* [C](#c) -* [C\#](#c-1) -* [C\+\+](#c-2) -* [Clojure](#clojure) -* [CoffeeScript](#coffeescript) -* [CSS](#css) -* [Go](#go) -* [Haskell](#haskell) -* [HTML](#html) -* [Java](#java) -* [JavaScript](#javascript) -* [Lua](#lua) -* [MATLAB](#matlab) -* [Objective\-C](#objective-c) -* [Perl](#perl) -* [PHP](#php) -* [Python](#python) -* [R](#r) -* [Ruby](#ruby) -* [Scala](#scala) -* [Shell](#shell) -* [Swift](#swift) -* [TeX](#tex) -* [Vim script](#vim-script) diff --git a/source/process.py b/source/process.py index e5962b5bd..17228b26b 100644 --- a/source/process.py +++ b/source/process.py @@ -2,27 +2,66 @@ from datetime import datetime import os import pandas as pd -from common import get_api_repos, get_graphql_data -from writefile import write_text, write_head_contents, write_ranking_repo +from common import get_api_repos, get_graphql_data, write_text, write_ranking_repo +import inspect -class Processor(object): +# languages = ['Python'] # For test +# languages_md = ['Python'] # For test +# table_of_contents = """ +# * [Python](#python)""" # For test +languages = ["ActionScript", "C", "CSharp", "CPP", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", + "JavaScript", "Lua", "MATLAB", "Objective-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", + "Swift", "TeX", "Vim-script"] +# Escape characters in markdown like # + - etc +languages_md = ["ActionScript", "C", "C\#", "C\+\+", "Clojure", "CoffeeScript", "CSS", "Go", "Haskell", "HTML", "Java", + "JavaScript", "Lua", "MATLAB", "Objective\-C", "Perl", "PHP", "Python", "R", "Ruby", "Scala", "Shell", + "Swift", "TeX", "Vim script"] +table_of_contents = """ +* [ActionScript](#actionscript) +* [C](#c) +* [C\#](#c-1) +* [C\+\+](#c-2) +* [Clojure](#clojure) +* [CoffeeScript](#coffeescript) +* [CSS](#css) +* [Go](#go) +* [Haskell](#haskell) +* [HTML](#html) +* [Java](#java) +* [Kotlin](#kotlin) +* [JavaScript](#javascript) +* [Lua](#lua) +* [MATLAB](#matlab) +* [Objective\-C](#objective-c) +* [Perl](#perl) +* [PHP](#php) +* [Python](#python) +* [R](#r) +* [Ruby](#ruby) +* [Scala](#scala) +* [Shell](#shell) +* [Swift](#swift) +* [TeX](#tex) +* [Vim script](#vim-script)""" + + +class ProcessorREST(object): + """ + Github REST API v3 + Deprecating API authentication through query parameters, should send the token in the header + ref: https://developer.github.com/changes/2020-02-10-deprecating-auth-through-query-param/ + search limit 10 times per minute, with token is 30 per minute + check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + """ def __init__(self): - # self.languages = ['Python'] # For test - # self.languages_md = ['Python'] # For test - self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] - self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] - # search limit 10 times per minute, with token is 30 per minute - # check rate_limit with : curl -H "Authorization: token your-access-token" https://api.github.com/rate_limit + self.api_repo_stars = r'https://api.github.com/search/repositories?q=stars:>0&sort=stars&per_page=100' self.api_repo_forks = r'https://api.github.com/search/repositories?q=forks:>0&sort=forks&per_page=100' self.api_repo_stars_lang = r'https://api.github.com/search/repositories?q=language:{lang}&stars:>0&sort=stars&per_page=100' - self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() def get_all_repos(self): @@ -35,80 +74,22 @@ class Processor(object): repos_forks = get_api_repos(self.api_repo_forks) repos_languages = {} - for lang in self.languages: + for lang in languages: print("Get most stars repos of {}...".format(lang)) - repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang = lang)) + repos_languages[lang] = get_api_repos(self.api_repo_stars_lang.format(lang=lang)) return repos_stars, repos_forks, repos_languages - def write_head_contents(self): - write_head_contents('../README.md') - - def write_readme_lang_md(self): - # Most stars save - write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') - write_ranking_repo('../README.md','a',self.repos_stars[0:10]) - print("Save most stars in README.md!") - os.makedirs('../Top100',exist_ok=True) - write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') - write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) - print("Save most stars in Top100/Top-100-stars.md!\n") - - # Most forks save - write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") - write_ranking_repo('../README.md','a',self.repos_forks[0:10]) - print("Save most forks in README.md!") - write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') - write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) - print("Save most forks in Top100/Top-100-forks.md!\n") - - # Most stars in language save - for i in range(len(self.languages)): - lang = self.languages[i] - write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) - write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) - print("Save most stars of {} in README.md!".format(lang)) - write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) - write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) - print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) - - def repo_to_df(self,repos,item): - # prepare for saving data to csv file - repos_list = [] - for idx, repo in enumerate(repos): - repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] - repos_list.append(repo_info) - return pd.DataFrame(repos_list,columns = self.col) - - def save_to_csv(self): - # save top100 repos info to csv file in Data/github-ranking-year-month-day.md - df_all = pd.DataFrame(columns=self.col) - df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') - df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') - df_all = df_all.append(df_repos_stars,ignore_index = True) - df_all = df_all.append(df_repos_forks,ignore_index = True) - for lang in self.repos_languages.keys(): - df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) - df_all = df_all.append(df_repos_lang,ignore_index = True) - - save_date = datetime.utcnow().strftime("%Y-%m-%d") - os.makedirs('../Data',exist_ok=True) - df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') - print('Save data to Data/github-ranking-'+save_date+'.csv') class ProcessorGQL(object): + """ + Github GraphQL API v4 + ref: https://docs.github.com/en/graphql + use graphql to get data, limit 5000 points per hour + check rate_limit with : + curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql + """ def __init__(self): - # self.languages = ["ActionScript","C","CSharp","CPP"] # For test - # self.languages_md = ["ActionScript","C","C\#","CPP"] # For test - self.languages = ["ActionScript","C","CSharp","CPP","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim-script"] - self.languages_md = ["ActionScript","C","C\#","C\+\+","Clojure","CoffeeScript","CSS","Go", - "Haskell","HTML","Java","JavaScript","Lua","MATLAB","Objective\-C","Perl", - "PHP","Python","R","Ruby","Scala","Shell","Swift","TeX","Vim script"] - # use graphql to get data, limit 5000 points per hour - # check rate_limit with : - # curl -H "Authorization: bearer your-access-token" -X POST -d "{\"query\": \"{ rateLimit { limit cost remaining resetAt used }}\" }" https://api.github.com/graphql self.gql_format = """query{ search(query: "%s", type: REPOSITORY, first: 100) { edges { @@ -138,14 +119,15 @@ class ProcessorGQL(object): } } """ - self.gql_stars = self.gql_format % ("stars:>1000 sort:stars") - self.gql_forks = self.gql_format % ("forks:>1000 sort:forks") - self.gql_stars_lang = self.gql_format % ("language:%s stars:>0 sort:stars") + self.gql_stars = self.gql_format % "stars:>1000 sort:stars" + self.gql_forks = self.gql_format % "forks:>1000 sort:forks" + self.gql_stars_lang = self.gql_format % "language:%s stars:>0 sort:stars" - self.col = ['rank','item','repo_name','stars','forks','language','repo_url','username','issues','last_commit','description'] - self.repos_stars, self.repos_forks, self.repos_languages = self.get_all_repos() + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] - def parse_gql_result(self,result): + @staticmethod + def parse_gql_result(result): res = [] for repo in result["data"]["search"]["edges"]: repo_data = repo['node'] @@ -169,70 +151,131 @@ class ProcessorGQL(object): print("Get repos of most stars...") repos_stars_gql = get_graphql_data(self.gql_stars) repos_stars = self.parse_gql_result(repos_stars_gql) + print("Get repos of most stars success!") print("Get repos of most forks...") repos_forks_gql = get_graphql_data(self.gql_forks) repos_forks = self.parse_gql_result(repos_forks_gql) + print("Get repos of most forks success!") repos_languages = {} - for lang in self.languages: + for lang in languages: print("Get most stars repos of {}...".format(lang)) repos_languages[lang] = self.parse_gql_result( - get_graphql_data(self.gql_stars_lang % (lang)) + get_graphql_data(self.gql_stars_lang % lang) ) + print("Get most stars repos of {} success!".format(lang)) return repos_stars, repos_forks, repos_languages - def write_head_contents(self): - write_head_contents('../README.md') + +class WriteFile(object): + def __init__(self, repos_stars, repos_forks, repos_languages): + self.repos_stars = repos_stars + self.repos_forks = repos_forks + self.repos_languages = repos_languages + self.col = ['rank', 'item', 'repo_name', 'stars', 'forks', 'language', 'repo_url', 'username', 'issues', + 'last_commit', 'description'] + self.repo_list = [] + self.repo_list.extend([{ + "desc": "Stars", + "desc_md": "Stars", + "title_readme": "Most Stars", + "title_100": "Top 100 Stars", + "file_100": "Top-100-stars.md", + "data": repos_stars, + "item": "top-100-stars", + }, { + "desc": "Forks", + "desc_md": "Forks", + "title_readme": "Most Forks", + "title_100": "Top 100 Forks", + "file_100": "Top-100-forks.md", + "data": repos_forks, + "item": "top-100-forks", + }]) + for i in range(len(languages)): + lang = languages[i] + lang_md = languages_md[i] + self.repo_list.append({ + "desc": "Forks", + "desc_md": "Forks", + "title_readme": lang_md, + "title_100": f"Top 100 Stars in {lang_md}", + "file_100": f"{lang}.md", + "data": repos_languages[lang], + "item": lang, + }) + + @staticmethod + def write_head_contents(): + # write the head and contents of README.md + write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + head_contents = inspect.cleandoc("""[Github Ranking](./README.md) + ========== + + **A list of the most github stars and forks repositories.** + + *Last Automatic Update Time: {write_time}* + + ## Table of Contents + + * [Most Stars](#most-stars) + * [Most Forks](#most-forks)""".format(write_time=write_time)) + table_of_contents + write_text("../README.md", 'w', head_contents) def write_readme_lang_md(self): - # Most stars save - write_text('../README.md','a','\n## Most Stars\n\nThis is top 10 list, for more click **[Github Top 100 Stars](Top100/Top-100-stars.md)**\n\n') - write_ranking_repo('../README.md','a',self.repos_stars[0:10]) - print("Save most stars in README.md!") - os.makedirs('../Top100',exist_ok=True) - write_text('../Top100/Top-100-stars.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Stars\n\n') - write_ranking_repo('../Top100/Top-100-stars.md','a',self.repos_stars) - print("Save most stars in Top100/Top-100-stars.md!\n") + os.makedirs('../Top100', exist_ok=True) + for repo in self.repo_list: + # README.md + title_readme, title_100, file_100, data = repo["title_readme"], repo["title_100"], repo["file_100"], repo["data"] + write_text('../README.md', 'a', + f"\n## {title_readme}\n\nThis is top 10, for more click **[{title_100}](Top100/{file_100})**\n\n") + write_ranking_repo('../README.md', 'a', data[:10]) + print(f"Save {title_readme} in README.md!") - # Most forks save - write_text("../README.md",'a',"## Most Forks\n\nThis is top 10 list, for more click **[Github Top 100 Forks](Top100/Top-100-forks.md)**\n\n") - write_ranking_repo('../README.md','a',self.repos_forks[0:10]) - print("Save most forks in README.md!") - write_text('../Top100/Top-100-forks.md','w','[Github Ranking](../README.md)\n==========\n\n## Github Top 100 Forks\n\n') - write_ranking_repo('../Top100/Top-100-forks.md','a',self.repos_forks) - print("Save most forks in Top100/Top-100-forks.md!\n") + # Top 100 file + write_text(f"../Top100/{file_100}", "w", + f"[Github Ranking](../README.md)\n==========\n\n## {title_100}\n\n") + write_ranking_repo(f"../Top100/{file_100}", 'a', data) + print(f"Save {title_100} in Top100/{file_100}!\n") - # Most stars in language save - for i in range(len(self.languages)): - lang = self.languages[i] - write_text('../README.md','a',"## {}\n\nThis is top 10 list, for more click **[Top 100 Stars in {}](Top100/{}.md)**\n\n".format(self.languages_md[i],self.languages_md[i],lang)) - write_ranking_repo('../README.md','a',self.repos_languages[lang][0:10]) - print("Save most stars of {} in README.md!".format(lang)) - write_text('../Top100/'+lang+'.md','w',"[Github Ranking](../README.md)\n==========\n\n## Top 100 Stars in {}\n\n".format(self.languages_md[i])) - write_ranking_repo('../Top100/'+lang+'.md','a',self.repos_languages[lang]) - print("Save most stars of {} in Top100/{}.md!\n".format(lang,lang)) - - def repo_to_df(self,repos,item): - # prepare for saving data to csv file + def repo_to_df(self, repos, item): + # prepare for saving data to csv file repos_list = [] for idx, repo in enumerate(repos): - repo_info = [idx + 1,item,repo['name'],repo['stargazers_count'],repo['forks_count'],repo['language'],repo['html_url'],repo['owner']['login'],repo['open_issues_count'],repo['pushed_at'],repo['description']] + repo_info = [idx + 1, item, repo['name'], repo['stargazers_count'], repo['forks_count'], repo['language'], + repo['html_url'], repo['owner']['login'], repo['open_issues_count'], repo['pushed_at'], + repo['description']] repos_list.append(repo_info) - return pd.DataFrame(repos_list,columns = self.col) + return pd.DataFrame(repos_list, columns=self.col) def save_to_csv(self): - # save top100 repos info to csv file in Data/github-ranking-year-month-day.md + # save top100 repos info to csv file in Data/github-ranking-year-month-day.md df_all = pd.DataFrame(columns=self.col) - df_repos_stars = self.repo_to_df(self.repos_stars,'top-100-stars') - df_repos_forks = self.repo_to_df(self.repos_forks,'top-100-forks') - df_all = df_all.append(df_repos_stars,ignore_index = True) - df_all = df_all.append(df_repos_forks,ignore_index = True) - for lang in self.repos_languages.keys(): - df_repos_lang = self.repo_to_df(self.repos_languages[lang],lang) - df_all = df_all.append(df_repos_lang,ignore_index = True) + for repo in self.repo_list: + df_repos = self.repo_to_df(repos=repo["data"], item=repo["item"]) + df_all = df_all.append(df_repos, ignore_index=True) save_date = datetime.utcnow().strftime("%Y-%m-%d") - os.makedirs('../Data',exist_ok=True) - df_all.to_csv('../Data/github-ranking-'+save_date+'.csv',index=False,encoding='utf-8') - print('Save data to Data/github-ranking-'+save_date+'.csv') \ No newline at end of file + os.makedirs('../Data', exist_ok=True) + df_all.to_csv('../Data/github-ranking-' + save_date + '.csv', index=False, encoding='utf-8') + print('Save data to Data/github-ranking-' + save_date + '.csv') + + +def run_by_gql(): + ROOT_PATH = os.path.abspath(os.path.join(__file__, "../../")) + os.chdir(os.path.join(ROOT_PATH, 'source')) + + # processor = ProcessorREST() # use Github REST API v3 + processor = ProcessorGQL() # use Github GraphQL API v4 + repos_stars, repos_forks, repos_languages = processor.get_all_repos() + wt_obj = WriteFile(repos_stars, repos_forks, repos_languages) + wt_obj.write_head_contents() + wt_obj.write_readme_lang_md() + wt_obj.save_to_csv() + + +if __name__ == "__main__": + t1 = datetime.now() + run_by_gql() + print("Total time: {}s".format((datetime.now() - t1).total_seconds())) diff --git a/source/run.py b/source/run.py deleted file mode 100644 index 2003ba6ce..000000000 --- a/source/run.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime -import os -from process import Processor,ProcessorGQL - -if __name__=="__main__": - t1 = datetime.now() - - ROOT_PATH = os.path.abspath(os.path.join(__file__,"../../")) - os.chdir(os.path.join(ROOT_PATH,'source')) - - # processor = Processor() # use Github REST API v3 - processor = ProcessorGQL() # use Github GraphQL API v4 - print("Write head and contents of README.md!") - processor.write_head_contents() - print("write to readme and languages.md") - processor.write_readme_lang_md() - print("Save data to csv file") - processor.save_to_csv() - - print("Total time: {}s".format((datetime.now()-t1).total_seconds())) \ No newline at end of file diff --git a/source/writefile.py b/source/writefile.py deleted file mode 100644 index aeb50470c..000000000 --- a/source/writefile.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime - -def write_text(file_name,method,text): - ''' - write text to file - - method: 'a'-append, 'w'-overwrite - ''' - with open(file_name, method, encoding='utf-8') as f: - f.write(text) - -def write_head_contents(file_name): - # write the head and contents of README.md - # write_head_contents('README.md') - write_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - head = "*Last Automatic Update Time: {}*\n\n".format(write_time) - with open('contents.md','r') as f: - contents = f.read() - write_text(file_name,'w',head+contents) - -def write_ranking_repo(file_name,method,repos): - # method: 'a'-append or 'w'-overwrite - table_head = "| Ranking | Project Name | Stars | Forks | Language | Open Issues | Description | Last Commit |\n\ -| ------- | ------------ | ----- | ----- | -------- | ----------- | ----------- | ----------- |\n" - with open(file_name, method, encoding='utf-8') as f: - f.write(table_head) - for idx, repo in enumerate(repos): - repo_description = repo['description'] - if repo_description is not None: - repo_description = repo_description.replace('|','\|') #in case there is '|' in description - f.write('| {} | [{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format( - idx + 1, - repo['name'], - repo['html_url'], - repo['stargazers_count'], - repo['forks_count'], - repo['language'], - repo['open_issues_count'], - repo_description, - repo['pushed_at'] - ) - ) - f.write('\n') \ No newline at end of file