Rewrote and refactored validator.js

Author: linusjf

spider5/cmdconfig.js

@@ -5,57 +5,57 @@
 
 function fullURL(url)
 {
-	if (!url.startsWith('http'))
-		return 'http://' + url;
+	if (!url.startsWith("http"))
+		return "http://" + url;
 	return url;
 }
 
 const cmdOptions = [
-  { name: 'help',
-    description: 'Display usage guide.',
-    alias: 'h',
+  { name: "help",
+    description: "Display usage guide.",
+    alias: "h",
     type: Boolean,
-    group: 'main'},
-	{ name: 'nesting',
+    group: "main"},
+	{ name: "nesting",
 		type: Number,
-		alias: 'n',
-		group: 'main',
-	description: 'Depth to which the starting url is to be traversed'},
-  { name: 'concurrency', 
+		alias: "n",
+		group: "main",
+	description: "Depth to which the starting url is to be traversed"},
+  { name: "concurrency", 
 		type: Number,
-		alias: 'c',
-		group: 'main',
-	description: 'Number of concurrent requests' },
-	{ name: 'url',
+		alias: "c",
+		group: "main",
+	description: "Number of concurrent requests" },
+	{ name: "url",
 		type: url => fullURL(url),
-		alias: 'u',
+		alias: "u",
 		defaultOption: true,
-		group: 'main',
-	description:'Url to be traversed'}
+		group: "main",
+	description:"Url to be traversed"}
 ]; 
 
 
 const sections = [
   {
-    header: 'Webcrawler app',
-    content: 'Crawls the url provided and downloads page links recursively to the nesting level specified.'
+    header: "Webcrawler app",
+    content: "Crawls the url provided and downloads page links recursively to the nesting level specified."
   },
   {
-    header: 'Main options',
+    header: "Main options",
     optionList: cmdOptions,
-    group: [ 'main' ]
+    group: [ "main" ]
   },
   {
-    header: 'Misc',
+    header: "Misc",
     optionList: cmdOptions,
-    group: '_none'
+    group: "_none"
   }
 ];
 
-const config = require('config');
-const commandLineArgs = require('command-line-args');
+const config = require("config");
+const commandLineArgs = require("command-line-args");
 const options = commandLineArgs(cmdOptions);
-const commandLineUsage = require('command-line-usage');
+const commandLineUsage = require("command-line-usage");
 const usage = commandLineUsage(sections);
 // The command line options
 module.exports.options = options;

spider5/spider.js

@@ -4,25 +4,23 @@
 /*jshint latedef: false */
 "use strict";
 
-const request = require('request');
-const fs = require('fs');
-const mkdirp = require('mkdirp');
-const path = require('path');
-const utilities = require('./utilities');
-const cmdConfig = require('./cmdconfig');
-const TaskQueue = require('./taskQueue');
-const validator = require('./validator');
+const request = require("request");
+const fs = require("fs");
+const mkdirp = require("mkdirp");
+const path = require("path");
+const utilities = require("./utilities");
+const cmdConfig = require("./cmdconfig");
+const TaskQueue = require("./taskQueue");
+const validator = require("./validator");
 
 function spiderLinks(currentUrl, body, nesting, callback) {
-  if(nesting === 0) {
+  if(nesting === 0) 
     return process.nextTick(callback);
-  }
 
   const links = utilities.getPageLinks(currentUrl, body);
-  if(links.length === 0) {
+  if(links.length === 0) 
     return process.nextTick(callback);
-  }
-let downloadQueue = new TaskQueue(cmdConfig.get('concurrency',2));
+let downloadQueue = new TaskQueue(cmdConfig.get("concurrency",2));
 
   let completed = 0, hasErrors = false;
   links.forEach(link => {
@@ -74,9 +72,9 @@ function spider(url, nesting, callback) {
   spidering.set(url, true);
 
   const filename = utilities.urlToFilename(url);
-  fs.readFile(filename, 'utf8', function(err, body) {
+  fs.readFile(filename, "utf8", function(err, body) {
     if(err) {
-      if(err.code !== 'ENOENT') {
+      if(err.code !== "ENOENT") {
         return callback(err);
       }
 
@@ -93,14 +91,24 @@ function spider(url, nesting, callback) {
   });
 }
 
-if (!validator.validate())
-	process.exit();
+const errors = validator.validate();
+if (errors.length || cmdConfig.get("help"))
+{
+  console.log(cmdConfig.usage);
+  errors.forEach((err) => 
+  {
+    console.error(err);
+  });
+  process.exit(errors.length);
+}
 
-spider(cmdConfig.get('url'), cmdConfig.get('nesting',1), (err) => {
+spider(cmdConfig.get("url"), cmdConfig.get("nesting",1), (err) => {
   if(err) {
-    console.log(err);
-    process.exit();
+    console.error(err);
+    process.exit(1);
   } else {
-    console.log('Download complete');
+    console.log("Download complete");
+    process.exit(0);
   }
 });
+

spider5/utilities.js

@@ -3,25 +3,25 @@
 /*jshint esversion: 6*/
 "use strict";
 
-const urlParse = require('url').parse;
-const urlResolve = require('url').resolve;
-const slug = require('slug');
-const path = require('path');
-const cheerio = require('cheerio');
+const urlParse = require("url").parse;
+const urlResolve = require("url").resolve;
+const slug = require("slug");
+const path = require("path");
+const cheerio = require("cheerio");
 
 module.exports.urlToFilename = function urlToFilename(url) {
   const parsedUrl = urlParse(url);
-  const urlPath = parsedUrl.path.split('/')
+  const urlPath = parsedUrl.path.split("/")
     .filter(function(component) {
-      return component !== '';
+      return component !== "";
     })
     .map(function(component) {
       return slug(component);
     })
-    .join('/');
+    .join("/");
 	let filename = path.join(parsedUrl.hostname, urlPath);
   if(!path.extname(filename).match(/htm/)) {
-    filename += '.html';
+    filename += ".html";
   }
   return filename;
 };
@@ -37,7 +37,7 @@ module.exports.getLinkUrl = function getLinkUrl(currentUrl, element) {
 };
 
 module.exports.getPageLinks = function getPageLinks(currentUrl, body) {
-  return [].slice.call(cheerio.load(body)('a'))
+  return [].slice.call(cheerio.load(body)("a"))
     .map(function(element) {
       return module.exports.getLinkUrl(currentUrl, element);
     })

spider5/validator.js

@@ -2,66 +2,67 @@
 /*jshint node: true */
 /*jshint esversion: 6 */
 "use strict";
-const validator = require('validator');
-const cmdConfig = require('./cmdconfig');
-const assert = require('assert');
+const validator = require("validator");
+const cmdConfig = require("./cmdconfig");
+const assert = require("assert");
 
-module.exports.validate = function()
-{
-	const options = cmdConfig.options;
-	let assertCount = 0;
-	function inc()
-	{
-		assertCount++;
-		return assertCount;
-	}
-	try{
-assert(options._all.url,'No url specified');
-	}
-	catch(err)
-	{
-		inc();
-		console.error(err.message);
-	}
-	if (options._all.url)
-	{
-	try{
-		assert(validator.isURL(options._all.url,{protocols:['http','https'],require_host: true, require_valid_protocol:true,require_protocols:true}),options._all.url + ' is invalid.');
-	}
-	catch(err)
-	{
-		inc();
-		console.error(err.message);
-	}
-	}
-	if (options._all.concurrency !== undefined)
-		try
-	{
-		assert(validator.isInt(options._all.concurrency.toString(),{min:1}),'Concurrency must be greater than 0');
-	}
-	catch(err)
-	{
-		inc();
-		console.error(err.message);
-	}
+module.exports.validate = function() {
+    const options = cmdConfig.options;
+    let errors = [];
+    errors = validateEmptyURL(options._all.url, errors);
+    errors = validateURLFormat(options._all.url, errors);
+    errors = validateConcurrency(options._all.concurrency, errors);
+    errors = validateNesting(options._all.nesting, errors);
+    return errors;
+};
 
-		if (options._all.nesting !== undefined)
-	{
-		try {
-		assert(validator.isInt(options._all.nesting.toString(),{gt:0}),'Nesting must be greater than 0');
-		}
-	catch(err)
-	{
-		inc();
-		console.error(err.message);
-	}
-	}
+function validateEmptyURL(url, errors) {
+    try {
+        assert(url, "No url specified");
+    } catch (err) {
+        errors.push(err.message);
+    }
+    return errors;
+}
 
-			if (assertCount || options._all.help)
-	{
-		console.error(cmdConfig.usage);
-		return false;
-	}
-	
-	return true;
-};
+function validateURLFormat(url, errors) {
+    if (url) {
+        try {
+            assert(validator.isURL(url, {
+                protocols: ["http", "https"],
+                require_host: true,
+                require_valid_protocol: true,
+                require_protocols: true
+            }), url + " is invalid.");
+        } catch (err) {
+            errors.push(err.message);
+        }
+    }
+    return errors;
+}
+
+function validateConcurrency(concurrency, errors) {
+    if (concurrency !== undefined) {
+        try {
+            assert(validator.isInt(concurrency.toString(), {
+                min: 1
+            }), "Concurrency must be greater than 0");
+        } catch (err) {
+            errors.push(err.message);
+        }
+    }
+    return errors;
+}
+
+function validateNesting(nesting, errors) {
+    if (nesting !== undefined) {
+        try {
+            assert(validator.isInt(nesting.toString(), {
+                gt: 0
+            }), "Nesting must be greater than 0");
+        } catch (err) {
+            errors.push(err.message);
+        }
+    }
+    return errors;
+}