Implement maxDepth option

Author: s0ph1e

lib/scraper.js

@@ -10,25 +10,18 @@ var _ = require('underscore');
 
 var defaults = require('./config/defaults');
 var types = require('./config/resource-types');
-var resursiveSources = require('./config/recursive-sources');
+var recursiveSources = require('./config/recursive-sources');
 var utils = require('./utils.js');
 var request = require('./request');
 var Resource = require('./resource');
-var loadHtml = require('./file-handlers/html');
-var loadCss = require('./file-handlers/css');
 var compareUrls = require('compare-urls');
 
-function getHandleFunction (resource) {
-	var type = resource.getType();
-	switch (type) {
-		case types.css: return loadCss;
-		case types.html: return function loadHtmlAndCss (context, po) {
-			return loadHtml(context, po).then(function (loaded) {
-				return loadCss(context, loaded);
-			});
-		};
-		default: return _.noop;
-	}
+var loadHtml = require('./file-handlers/html');
+var loadCss = require('./file-handlers/css');
+function loadHtmlAndCss (context, po) {
+	return loadHtml(context, po).then(function (loaded) {
+		return loadCss(context, loaded);
+	});
 }
 
 function Scraper (options) {
@@ -89,6 +82,20 @@ Scraper.prototype.getDirectoryByExtension = function getDirectoryByExtension (ex
 		.value() || '';
 };
 
+Scraper.prototype.getResourceHandler = function getHandler (resource) {
+	var self = this;
+	var type = resource.getType();
+	var depth = resource.getDepth();
+	var depthGreaterThanMax = self.options.maxDepth && depth > self.options.maxDepth;
+
+	switch (true) {
+		case depthGreaterThanMax: return _.noop;
+		case type == types.css: return loadCss;
+		case type == types.html: return loadHtmlAndCss;
+		default: return _.noop;
+	}
+};
+
 Scraper.prototype.loadResource = function loadResource (resource) {
 	var self = this;
 
@@ -108,7 +115,7 @@ Scraper.prototype.loadResource = function loadResource (resource) {
 		return self.makeRequest(url).then(function requestCompleted(data) {
 			resource.setUrl(data.url);  // Url may be changed in redirects
 			resource.setText(data.body);
-			handleFile = getHandleFunction(resource);
+			handleFile = self.getResourceHandler(resource);
 			return handleFile(self, resource);
 		}).then(function fileHandled() {
 			var filename = path.join(self.options.directory, resource.getFilename());
@@ -143,7 +150,7 @@ Scraper.prototype.prepare = function prepare () {
 	});
 
 	if (self.options.recursive) {
-		self.options.sources = _.union(self.options.sources, resursiveSources);
+		self.options.sources = _.union(self.options.sources, recursiveSources);
 	}
 
 	return ensureDirAsync(self.options.directory);

test/unit/scraper-test.js

@@ -551,6 +551,10 @@ describe('Scraper', function () {
 		});
 	});
 
+	describe('#getResourceHandler', function() {
+
+	});
+
 	describe('#scrape', function() {
 		it('should call methods in sequence', function(done) {
 			nock('http://example.com').get('/').reply(200, 'OK');