New parsing method issues fixed

Author: Zakaria RACHEDI

lib/app.js

@@ -76,6 +76,7 @@ function getPageLinks() {
 
 function getItems(category, links) {
 	gi.getItems(category, links, function(items){
+		if(fs.existsSync('./data/links/resume.json')) fs.unlinkSync('./data/links/resume.json');
 		category = category.replace(/ /g,'');
 		fsPath.writeFile('./data/' + category + '.json', JSON.stringify(items), function(err){
 			if (err) console.log(err);

lib/getItems.js

@@ -15,8 +15,8 @@ var CLI = require('clui'),
 var currentPosition = 0;
 var progressbarPosition = 0;
 var itemsList = [];
-var fullList;
-const parserLimit = 700; 
+var gCategory;
+var gLinks; 
 var thisProgressBar = new Progress(20);
 var options = {
     method: 'POST',
@@ -27,15 +27,15 @@ var options = {
 };
 
 var getItems = exports.getItems = function(category, links, back) {
+	gCategory = category;
+	gLinks = links;
 	if (currentPosition == 0) {
 		if(fs.existsSync('./data/links/resume.json')) {
 			var itemsResumed = JSON.parse(fs.readFileSync('./data/'+category+'.json', 'utf8'));
 			itemsList = itemsResumed;
 		}
-		// console.log('initiale itemsList: '+itemsList.length);
-		// console.log('initiale links: '+links.length);
 	}
-	fullList = itemsList.length + links.length;
+	// if(currentPosition == 10) limitCrossed(gCategory, currentPosition, gLinks, itemsList);
 	getData(links[currentPosition],back, function(item){
 		itemsList.push(item);
 		if (progressbarPosition >= links.length * 0.05) {
@@ -44,14 +44,8 @@ var getItems = exports.getItems = function(category, links, back) {
 		}
 		progressbarPosition++;
 		currentPosition++;
-        if(currentPosition == parserLimit) {
-			// console.log('itemList: '+itemsList.length+' links: '+fullList);
-			if(itemsList.length !== fullList) limitCrossed(category, currentPosition, links, itemsList);
-		}else if(currentPosition < links.length) getItems(category, links, back); // any more items in array?
-		else {
-        	console.log(thisProgressBar.update(100, 100));
-			back(itemsList);
-		}
+		if(currentPosition < links.length) getItems(category, links, back); // any more items in array?
+		else console.log(thisProgressBar.update(100, 100)), back(itemsList);
 	});
 };
 
@@ -111,8 +105,8 @@ function getData(url, back, callback) {
         	console.log('\x1b[33m%s\x1b[0m' ,'\n Error 404 detected ! Maybe empty item (Encyclopedia error).');
         	callback();
         }else if(err.statusCode == '429') {
-        	console.log('\x1b[31m%s\x1b[0m' ,'\n /!\\ Error 429 detected ! Too many request, be careful Ankama can ban your IP. /!\\ Never parse more than 20 pages/hour');
-        	process.exit();
+        	console.log('\x1b[31m%s\x1b[0m' ,'\n /!\\ Error 429 detected ! Too many request, be careful Ankama can ban your IP. /!\\ Never parse more than 700 items/hour');
+        	limitCrossed(gCategory, currentPosition, gLinks, itemsList);
         }else {
         	console.log(err);
         	console.log('\x1b[31m%s\x1b[0m' ,'/!\\Broken promise all');
@@ -123,15 +117,15 @@ function getData(url, back, callback) {
 
 function limitCrossed(category, lastIndex, links, items) {
 	if(fs.existsSync('./data/links/resume.json')) fs.unlinkSync('./data/links/resume.json');
-	links = links.slice(lastIndex, 10);
+	links = links.slice(lastIndex);
 	var resume = {'category': category, links: links};
 
 	category = category.replace(/ /g,'');
 	fsPath.writeFile('./data/' + category + '.json', JSON.stringify(items), function(err){
 		if (err) console.log(err);
 		fsPath.writeFile('./data/links/resume.json', JSON.stringify(resume), function(err){
 			if (err) console.log(err);
-			console.log('\x1b[31m%s\x1b[0m' ,'\n/!\\You reached maximum request per hour ('+parserLimit+'), over pass it will provoke a ban IP from Ankama. Resume the parsing after 1h !');
+			console.log('\x1b[31m%s\x1b[0m' ,'\n/!\\You reached maximum request per hour, over pass it will provoke a ban IP from Ankama. Resume the parsing after 1h !');
 			console.log('\x1b[33m%s\x1b[0m' ,'\n INFO : ' +items.length+ ' item(s) were crawled.');
 			console.log('\x1b[33m%s\x1b[0m' ,'/!\\Don\'t worry, the app will resume the parsing from last item parsed ;)');
 			setTimeout(process.exit(),4000);