Add read/write methods for archiving GitHub datasets

Author: nelsonni

minerutils/github.py

@@ -8,6 +8,7 @@
 import json
 import datetime
 import sys
+import bigjson
 from minerutils.auth import MinerWithAuthentication
 
 class GitHub(MinerWithAuthentication):
@@ -51,6 +52,8 @@ def _get(self, url, params={}, headers={}):
 		return resp
 
 	def _getNextURL(self, resp):
+		if (resp is None):
+			return None
 		if (not 'Link' in resp.headers):
 			return None
 		linksText = resp.headers['Link']
@@ -80,3 +83,21 @@ def repoExists(self, user, repo):
 			return False
 		else:
 			return True
+		
+	def writeData(self, path, data):
+		try:
+			with open(path, 'w', encoding='utf8') as f:
+				json.dump(data, f, ensure_ascii=False)
+		except IOError:
+			print("File not accessible")
+
+	def readData(self, path):
+		try:
+			wrapper = None
+			with open(path, 'rb') as f:
+				wrapper = bigjson.load(f, 'utf8').to_python()
+			return wrapper
+		except FileNotFoundError:
+			print("File not found")
+		except IOError:
+			print("File not accessible")

tests/test_github.py

@@ -1,4 +1,5 @@
 import unittest
+import os
 from minerutils import GitHub
 
 class GitHubTest(unittest.TestCase):
@@ -8,6 +9,8 @@ def setUp(self):
 
 	def tearDown(self):
 		self.g = None
+		if os.path.exists('sample.json'):
+			os.remove('sample.json')
 
 	def test_repo_exists(self):
 		self.assertTrue(self.g.repoExists("EPICLab", "EPICLab.github.io"))
@@ -44,5 +47,11 @@ def test_url_params(self):
 		users = self.g.get('/search/users?q=EPICLab')
 		self.assertTrue(len(users) > 0)
 
+	def test_data_io(self):
+		data = [{"1": "one"}, {"2": "two"}, {"3": "three"}]
+		self.g.writeData('sample.json', data)
+		output = self.g.readData('sample.json')
+		self.assertListEqual(data, output)
+
 if __name__ == '__main__':
 	unittest.main()