Merge pull request #41 from Nullifiers/feature/root-level-readme

Create README files at domain and root level also

Author: rajatgoyal715

.gitignore

@@ -16,7 +16,7 @@ __pycache__
 
 # Script results
 Hackerrank/
-metadata*
+metadata.json
 
 # Config file
 *.yaml

.travis.yml

@@ -8,4 +8,4 @@ install:
 
 # command to run tests
 script:
-  python -m unittest tests/*
+  python -m unittest tests/*.py

hsc/__init__.py

@@ -0,0 +1 @@
+from .crawler import Crawler

hsc/constants.py

@@ -0,0 +1,50 @@
+extensions = {
+	'ada': 'ada',
+	'bash': 'sh',
+	'c': 'c',
+	'clojure': 'clj',
+	'coffeescript': 'coffee',
+	'cpp': 'cpp',
+	'cpp14': 'cpp',
+	'csharp': 'cs',
+	'd': 'd',
+	'db2': 'sql',
+	'elixir': 'ex',
+	'erlang': 'erl',
+	'fortran': 'for',
+	'fsharp': 'fs',
+	'go': 'go',
+	'groovy': 'groovy',
+	'haskell': 'hs',
+	'java': 'java',
+	'java8': 'java',
+	'javascript': 'js',
+	'julia': 'jl',
+	'kotlin': 'kt',
+	'lolcode': 'lol',
+	'lua': 'lua',
+	'mysql': 'sql',
+	'objectivec': 'm',
+	'ocaml': 'ml',
+	'octave': 'oct',
+	'oracle': 'sql',
+	'pascal': 'pas',
+	'perl': 'pl',
+	'php': 'php',
+	'pypy': 'py',
+	'pypy3': 'py',
+	'python': 'py',
+	'python3': 'py',
+	'racket': 'rkt',
+	'r': 'r',
+	'ruby': 'rb',
+	'rust': 'rs',
+	'sbcl': 'lisp',
+	'scala': 'scala',
+	'swift': 'swift',
+	'smalltalk': 'st',
+	'tcl': 'tcl',
+	'tsql': 'sql',
+	'visualbasic': 'vbs',
+	'whitespace': 'hs',
+}

hsc/crawler.py

@@ -1,48 +1,29 @@
 import os
-import json
 import requests
 import getpass
 import configargparse
-from progress.bar import ChargingBar
-
-
-class CustomProgress(ChargingBar):
-	message = 'Downloading Solutions'
-	suffix = '%(percent)d%% [%(index)d/%(max)d]'
-
-
-class Metadata:
-
-	METADATA_FILE_NAME = 'metadata.json'
-
-	def __init__(self):
-		self.metadata = {}
-		if (os.path.isfile(self.METADATA_FILE_NAME)):
-			self.metadata = json.load(open(self.METADATA_FILE_NAME))
-
-	def put(self, challenge_id, submission_id):
-		self.metadata[str(challenge_id)] = str(submission_id)
-		json.dump(self.metadata, open(self.METADATA_FILE_NAME, 'w'))
-
-	def get(self, challenge_id):
-		challenge_id_string = str(challenge_id)
-		if challenge_id_string not in self.metadata:
-			self.metadata[challenge_id_string] = -1
-		submission_id_string = self.metadata[challenge_id_string]
-		return int(submission_id_string)
+from .progress_bar import CustomProgress
+from .metadata import Metadata
+from .constants import extensions
 
 
 class Crawler:
 	base_url = 'https://www.hackerrank.com/'
 	login_url = base_url + 'auth/login'
 	submissions_url = base_url + 'rest/contests/master/submissions/?offset={}&limit={}'
 	challenge_url = base_url + 'rest/contests/master/challenges/{}/submissions/{}'
-	domain_url = base_url + 'domains/{}/{}'
+	domain_url = base_url + 'domains/{}'
+	subdomain_url = base_url + 'domains/{}/{}'
 	problem_url = base_url + 'challenges/{}/problem'
 
-	new_readme_text = '## [{}]({})\n\n|Problem Name|Problem Link|Language|Solution Link|\n---|---|---|---\n'
-	readme_headers_len = len(new_readme_text.split('\n')) - 1
-	problem_readme_text = '|{}|[Problem]({})|{}|[Solution](./{})|\n'
+	subdomain_readme_text = '## [{}]({})\n\n|Problem Name|Problem Link|Language|Solution Link|\n---|---|---|---\n'
+	domain_readme_text = '## [{}]({})\n\n|Subdomain|Problem Name|Problem Link|Language|Solution Link|\n---|---|---|---|---\n'
+	root_readme_text = '## [Hackerrank]({})\n\n|Domain|Subdomain|Problem Name|Problem Link|Language|Solution Link|\n---|---|---|---|---|---\n'
+	readme_headers_len = len(subdomain_readme_text.split('\n')) - 1
+
+	subdomain_readme_row = '|{}|[Problem]({})|{}|[Solution]({})|\n'
+	domain_readme_row = '|{}|{}|[Problem]({})|{}|[Solution]({})|\n'
+	root_readme_row = '|{}|{}|{}|[Problem]({})|{}|[Solution]({})|\n'
 
 	base_folder_name = 'Hackerrank'
 
@@ -51,57 +32,7 @@ class Crawler:
 	# prepend language in file extension e.g Hackerrank/Regex/Introduction/matching.python3.py
 	prepend_language_in_extension = False
 
-	# file extensions
-	file_extensions = {
-		'ada': 'ada',
-		'bash': 'sh',
-		'c': 'c',
-		'clojure': 'clj',
-		'coffeescript': 'coffee',
-		'cpp': 'cpp',
-		'cpp14': 'cpp',
-		'csharp': 'cs',
-		'd': 'd',
-		'db2': 'sql',
-		'elixir': 'ex',
-		'erlang': 'erl',
-		'fortran': 'for',
-		'fsharp': 'fs',
-		'go': 'go',
-		'groovy': 'groovy',
-		'haskell': 'hs',
-		'java': 'java',
-		'java8': 'java',
-		'javascript': 'js',
-		'julia': 'jl',
-		'kotlin': 'kt',
-		'lolcode': 'lol',
-		'lua': 'lua',
-		'mysql': 'sql',
-		'objectivec': 'm',
-		'ocaml': 'ml',
-		'octave': 'oct',
-		'oracle': 'sql',
-		'pascal': 'pas',
-		'perl': 'pl',
-		'php': 'php',
-		'pypy': 'py',
-		'pypy3': 'py',
-		'python': 'py',
-		'python3': 'py',
-		'racket': 'rkt',
-		'r': 'r',
-		'ruby': 'rb',
-		'rust': 'rs',
-		'sbcl': 'lisp',
-		'scala': 'scala',
-		'swift': 'swift',
-		'smalltalk': 'st',
-		'tcl': 'tcl',
-		'tsql': 'sql',
-		'visualbasic': 'vbs',
-		'whitespace': 'hs',
-	}
+	file_extensions = extensions
 
 	def __init__(self):
 		self.session = requests.Session()
@@ -149,27 +80,74 @@ def store_submission(self, file_name, code):
 		with open(file_name, 'w') as text_file:
 			text_file.write(code)
 
-	def update_readme(self, readme_file_path, problem_readme_text):
+	def update_readme(self, readme_file_path, readme_text):
 		header_length = self.readme_headers_len
 		with open(readme_file_path, 'r+') as text_file:
 			lines = text_file.readlines()
-			lines.append(problem_readme_text)
+			lines.append(readme_text)
 			sortedlines = lines[:header_length] + sorted(lines[header_length:])
 			text_file.seek(0)
 			text_file.writelines(sortedlines)
 
-	def create_readme(self, track_name, track_url, file_name):
-		if track_name is not None:
-			os.makedirs(os.path.dirname(file_name), exist_ok=True)
-			text = self.new_readme_text.format(track_name, track_url)
-			with open(file_name, 'w') as text_file:
-				text_file.write(text)
-
-	def get_file_path(self, folder_name, file_name_with_extension):
-		return os.path.join(self.base_folder_name, folder_name, file_name_with_extension)
+	def write(self, file_name, text):
+		os.makedirs(os.path.dirname(file_name), exist_ok=True)
+		with open(file_name, 'w') as text_file:
+			text_file.write(text)
+
+
+	def create_readmes(self, domain_name, subdomain_name, domain_url, subdomain_url,
+				subdomain_readme_path, domain_readme_path, root_readme_path):
+		"""
+		Method to check if readme files already exist. If readme files doesn't exist, then create them and add headers.
+		"""
+		if not os.path.exists(subdomain_readme_path):
+			text = self.subdomain_readme_text.format(subdomain_name, subdomain_url)
+			self.write(subdomain_readme_path, text)
+
+		if not os.path.exists(domain_readme_path):
+			text = self.domain_readme_text.format(domain_name, domain_url)
+			self.write(domain_readme_path, text)
+
+		if not os.path.exists(root_readme_path):
+			text = self.root_readme_text.format(self.base_url)
+			self.write(root_readme_path, text)
+
+
+	def update_readmes(self, domain_name, subdomain_name, domain_url, subdomain_url,
+				challenge_name, challenge_slug, language, file_name_with_extension):
+		"""
+		Method to add a new row corresponding to a new solution in the readme files
+		"""
+		subdomain_readme_path = os.path.join(self.base_folder_name, domain_name, subdomain_name, 'README.md')
+		if self.make_language_folder:
+			subdomain_readme_path = os.path.join(self.base_folder_name, domain_name, subdomain_name, language, 'README.md')
+		domain_readme_path = os.path.join(self.base_folder_name, domain_name, 'README.md')
+		root_readme_path = os.path.join(self.base_folder_name, 'README.md')
+
+		self.create_readmes(domain_name, subdomain_name, domain_url, subdomain_url,
+				subdomain_readme_path, domain_readme_path, root_readme_path)
+
+		problem_url = self.problem_url.format(challenge_slug)
+		
+		file_path_relative_to_subdomain = './' + file_name_with_extension
+		file_path_relative_to_domain = '{}/{}'.format(subdomain_name, file_name_with_extension)
+		file_path_relative_to_root = '{}/{}/{}'.format(domain_name, subdomain_name, file_name_with_extension)
+		subdomain_readme_text = self.subdomain_readme_row.format(challenge_name, problem_url, language, file_path_relative_to_subdomain)
+		domain_readme_text = self.domain_readme_row.format(subdomain_name, challenge_name, problem_url, language, file_path_relative_to_domain)
+		root_readme_text = self.root_readme_row.format(domain_name, subdomain_name, challenge_name, problem_url, language, file_path_relative_to_root)
+		self.update_readme(
+			subdomain_readme_path,
+			subdomain_readme_text,
+		)
+		self.update_readme(
+			domain_readme_path,
+			domain_readme_text,
+		)
+		self.update_readme(
+			root_readme_path,
+			root_readme_text,
+		)
 
-	def get_readme_path(self, folder_name):
-		return os.path.join(self.base_folder_name, folder_name, 'README.md')
 
 	def get_submissions(self, submissions):
 		headers = self.headers
@@ -195,47 +173,49 @@ def get_submissions(self, submissions):
 				data = resp.json()['model']
 				code = data['code']
 				track = data['track']
-
-				folder_name = 'Others'
-				file_extension = '.' + language
+				
+				# Default should be empty
+				file_extension = ''
 				file_name = challenge_slug
-				track_folder_name = 'Others'
-				track_url = ''
 
-				if track:
-					track_folder_name = track['name'].strip().replace(' ', '')
-					track_url = self.domain_url.format(track['track_slug'], track['slug'])
-					parent_folder_name = track['track_name'].strip().replace(' ', '')
-					folder_name = os.path.join(parent_folder_name, track_folder_name)
+				domain_name = 'Others'
+				subdomain_name = 'Miscellaneous'
 
-				if self.make_language_folder:
-					folder_name = os.path.join(folder_name, language)
+				domain_slug = ''
+				subdomain_slug = ''
+
+				if track:
+					domain_name = track['track_name'].strip().replace(' ', '')
+					subdomain_name = track['name'].strip().replace(' ', '')
+					domain_slug = track['track_slug']
+					subdomain_slug = track['slug']
+				
+				domain_url = self.domain_url.format(domain_slug)
+				subdomain_url = self.subdomain_url.format(domain_slug, subdomain_slug)
 
 				if language in self.file_extensions:
-					if not self.prepend_language_in_extension:
-						file_extension = ''
+					if self.prepend_language_in_extension:
+						file_extension += '.{}'.format(language)
 					file_extension += '.{}'.format(self.file_extensions[language])
 
 				if file_extension.endswith('.java'):
 					file_name = challenge_name.replace(' ','')
 
-				file_path = self.get_file_path(folder_name, file_name + file_extension)
+				file_name_with_extension = file_name + file_extension
+				file_path = os.path.join(self.base_folder_name, domain_name, subdomain_name, file_name_with_extension)
+				if self.make_language_folder:
+					file_path = os.path.join(self.base_folder_name, domain_name, subdomain_name, language, file_name_with_extension)
 				self.store_submission(file_path, code)
-				readme_file_path = self.get_readme_path(folder_name)
-				if not os.path.exists(readme_file_path):
-					self.create_readme(track_folder_name, track_url, readme_file_path)
-				problem_url = self.problem_url.format(challenge_slug)
-				readme_text = self.problem_readme_text.format(challenge_name, problem_url, language, file_name + file_extension)
-				self.update_readme(
-					readme_file_path,
-					readme_text,
-				)
+				
+				self.update_readmes(domain_name, subdomain_name, domain_url, subdomain_url,
+						challenge_name, challenge_slug, language, file_name_with_extension)
+
 			progress.next()
 		progress.finish()
 		print('All Solutions Crawled')
 
-def main():
 
+def main():
 	crawler = Crawler()
 	crawler.parse_script()
 	if not crawler.authenticate():

hsc/metadata.py

@@ -0,0 +1,24 @@
+import os
+import json
+
+class Metadata:
+
+	METADATA_FILE_NAME = 'metadata.json'
+
+	def __init__(self):
+		self.metadata = {}
+		if os.path.isfile(self.METADATA_FILE_NAME):
+			with open(self.METADATA_FILE_NAME) as fp:
+				self.metadata = json.load(fp)
+
+	def put(self, challenge_id, submission_id):
+		self.metadata[str(challenge_id)] = str(submission_id)
+		with open(self.METADATA_FILE_NAME, 'w') as fp:
+			json.dump(self.metadata, fp)
+
+	def get(self, challenge_id):
+		challenge_id_string = str(challenge_id)
+		if challenge_id_string not in self.metadata:
+			self.metadata[challenge_id_string] = -1
+		submission_id_string = self.metadata[challenge_id_string]
+		return int(submission_id_string)

hsc/progress_bar.py

@@ -0,0 +1,4 @@
+from progress.bar import ChargingBar
+
+class CustomProgress(ChargingBar):
+	suffix = '%(percent)d%% [%(index)d/%(max)d]'

tests/test_crawler.py

@@ -1,10 +1,10 @@
 import unittest
-from hsc import crawler
+from hsc import Crawler
 
 class TestCrawler(unittest.TestCase):
 
 	def setUp(self):
-		self.crawler_obj = crawler.Crawler()
+		self.crawler_obj = Crawler()
 
 	def test_crawler_obj_is_not_none(self):
 		self.assertIsNotNone(self.crawler_obj)