Max items per category can be specify now froom CLI

Author: Zakaria RACHEDI

lib/app.js

@@ -12,6 +12,7 @@ var url;
 var globalUrl;
 var itemCategory;
 var maxItem = 1;
+var all = true;
 var requestOpts = {
     url: '',
     method: 'GET',
@@ -40,6 +41,7 @@ function crawlerInit (cmdResponse) {
 	var countdown = new Spinner('Crawler in progress... It could take some time ', ['⣾','⣽','⣻','⢿','⡿','⣟','⣯','⣷']);
 	countdown.start();
 	maxItem = cmdResponse.maxItem;
+	all = cmdResponse.all;
 	itemCategory = cmdResponse.category;
 	cmdResponse.language ==  'french' ? url = sw.cmdSwitch(itemCategory) : url = sw.cmdSwitchEn(itemCategory);
 	cmdResponse.game ==  'dofus' ? url = url : url = url.replace('https://www.dofus.com', 'https://www.dofus-touch.com');
@@ -52,35 +54,35 @@ function getPageLinks() {
 	return request(requestOpts).then(function ($) {
 		var links = [];
 		$('tbody').find('tr').each(function(i, tr){
-			if(i >= maxItem) return false;
+			if(!all) if(i >= maxItem) return false;
 			var link = globalUrl + $(this).find('td').eq(1).find('a').attr('href');
 			links.push(link);
 		});
 		return links;
 	}).then(function(links) {
-		fsPath.writeFile('./data/links/' + itemCategory + '_links.json', JSON.stringify(links), function(err){
-			if (err) console.log(err);
-			console.log('\x1b[36m%s\x1b[0m' ,'\n SUCCESS : all item(s) links crawled.');
-			console.log('\x1b[36m%s\x1b[0m' ,'\n START of item(s) crawling.');
-			getItems(itemCategory, getLinksFromFile());
-		});
+		fsPath.writeFileSync('./data/links/' + itemCategory + '_links.json', JSON.stringify(links));
+		console.log('\x1b[36m%s\x1b[0m' ,'\n SUCCESS : all item(s) links crawled.');
+		console.log('\x1b[36m%s\x1b[0m' ,'\n START of item(s) crawling.');
+		getItems(itemCategory, getLinksFromFile());
 	}).catch(function(err) {
-	    console.log('\x1b[31m%s\x1b[0m' ,'/!\\Broken promise from getPageLinks');
-	    console.log(err);
-		process.exit();
+		console.log('\x1b[31m%s\x1b[0m' ,'/!\\Broken promise from getPageLinks');
+	    if(err.statusCode == '429') console.log('\x1b[31m%s\x1b[0m' ,'\n/!\\Error 429 detected ! You reached maximum request per hour, over pass it will provoke a ban IP from Ankama. Resume the parsing after 1h !'), process.exit();
+        else if(err.code == 'ETIMEDOUT') console.log('\x1b[31m%s\x1b[0m' ,'\n /!\\ Error ETIMEDOUT detected ! Your connexion took too much time to respond.'), process.exit();
+        else if(err.message == 'Error: read ECONNRESET') console.log('\x1b[31m%s\x1b[0m' ,'\n /!\\ Error ECONNRESET detected ! Connexion shutdown or reset, verify your internet connexion !'), process.exit();
+        else if(err.message == 'Error: unable to verify the first certificate') console.log('\x1b[33m%s\x1b[0m' ,'/!\\However, don\'t worry: if your relaunch it, the app will resume the parsing from last item parsed ;)'), process.exit();
+        else console.log(err), console.log('\x1b[31m%s\x1b[0m' ,'/!\\Broken promise from getPageLinks'), process.exit();
 	  });
 }
 
 function getItems(category, links) {
 	gi.getItems(category, links, function(items){
-		if(fs.existsSync('./data/links/resume.json')) fs.unlinkSync('./data/links/resume.json');
 		category = category.replace(/ /g,'');
-		fsPath.writeFile('./data/' + category + '.json', JSON.stringify(items), function(err){
-			if (err) console.log(err);
-			console.log('\x1b[32m%s\x1b[0m' ,'\n SUCCESS : ' +items.length+ ' item(s) were crawled.');
-			console.log('\x1b[33m%s\x1b[0m' ,'File ' + category +'.json' + ' was generated under "data/" folder.');
-			process.exit();
-		});
+		if(fs.existsSync('./data/links/resume.json')) fs.unlinkSync('./data/links/resume.json');
+		if(fs.existsSync('./data/links/' + category + '_links.json')) fs.unlinkSync('./data/links/' + category + '_links.json');
+		fsPath.writeFileSync('./data/' + category + '.json', JSON.stringify(items));
+		console.log('\x1b[32m%s\x1b[0m' ,'\n SUCCESS : ' +items.length+ ' item(s) were crawled.');
+		console.log('\x1b[33m%s\x1b[0m' ,'File ' + category +'.json' + ' was generated under "data/" folder.');
+		process.exit();
 	});
 }
 

lib/cli-view/cmd.js

@@ -42,11 +42,19 @@ var equipment = [
         return val.toLowerCase().replace(/ /g,'');
       }
   },
+  {
+    type: 'confirm',
+    name: 'all',
+    message: 'Do you want to parse [ALL/MAX] items of the category ?'
+  },
   {
     type: 'input',
     name: 'maxItem',
     message: 'How many items do you want to parse ?',
     default: '1',
+    when: function(answers) {
+      return !answers.all;
+    },
     validate: function(value) {
       var valid = !isNaN(parseFloat(value));
       return valid || 'Please enter a number';
@@ -65,11 +73,19 @@ var weapon = [
         return val.toLowerCase().replace(/ /g,'');
       }
   },
+  {
+    type: 'confirm',
+    name: 'all',
+    message: 'Do you want to parse [ALL/MAX] items of the category ?'
+  },
   {
     type: 'input',
     name: 'maxItem',
     message: 'How many items do you want to parse ?',
     default: '1',
+    when: function(answers) {
+      return !answers.all;
+    },
     validate: function(value) {
       var valid = !isNaN(parseFloat(value));
       return valid || 'Please enter a number';
@@ -79,11 +95,19 @@ var weapon = [
 ];
 
 var page = [
+  {
+    type: 'confirm',
+    name: 'all',
+    message: 'Do you want to parse [ALL/MAX] items of the category ?'
+  },
   {
     type: 'input',
     name: 'maxItem',
     message: 'How many items do you want to parse ?',
     default: '1',
+    when: function(answers) {
+      return !answers.all;
+    },
     validate: function(value) {
       var valid = !isNaN(parseFloat(value));
       return valid || 'Please enter a number';

package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlit",
-  "version": "1.1.1",
+  "version": "1.2.1",
   "description": "Crawl dofus encyclopedia ",
   "main": "lib/app.js",
   "bin": "lib/app.js",