Implement recursive option

s0ph1e · s0ph1e · commit 0665e47e55b5 · 2016-01-01T23:28:31.000+02:00
diff --git a/lib/config/recursive-sources.js b/lib/config/recursive-sources.js
@@ -0,0 +1,3 @@
+module.exports = [
+	{ selector: 'a', attr: 'href' }
+];
diff --git a/lib/scraper.js b/lib/scraper.js
@@ -10,6 +10,7 @@ var _ = require('underscore');
 
 var defaults = require('./config/defaults');
 var types = require('./config/resource-types');
+var resursiveSources = require('./config/recursive-sources');
 var utils = require('./utils.js');
 var request = require('./request');
 var Resource = require('./resource');
@@ -141,6 +142,10 @@ Scraper.prototype.prepare = function prepare () {
 		return new Resource(url, filename);
 	});
 
+	if (self.options.recursive) {
+		self.options.sources = _.union(self.options.sources, resursiveSources);
+	}
+
 	return ensureDirAsync(self.options.directory);
 };
 
diff --git a/test/functional/mocks/recursive/about.html b/test/functional/mocks/recursive/about.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<a href="/link1.html"></a>
+<a href="/link2.html"></a>
+<a href="/link3.html"></a>
+
+</body>
+</html>
diff --git a/test/functional/mocks/recursive/index.html b/test/functional/mocks/recursive/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+<a href="/about.html"></a>
+
+</body>
+</html>
diff --git a/test/functional/recursive-test.js b/test/functional/recursive-test.js
@@ -0,0 +1,55 @@
+require('should');
+var nock = require('nock');
+var fs = require('fs-extra');
+var path = require('path');
+var _ = require('underscore');
+var scraper = require('../../index');
+
+var testDirname = __dirname + '/.recursive';
+var mockDirname = __dirname + '/mocks/recursive';
+
+describe('Functional recursive downloading', function() {
+
+	beforeEach(function() {
+		nock.cleanAll();
+		nock.disableNetConnect();
+	});
+
+	afterEach(function() {
+		nock.cleanAll();
+		nock.enableNetConnect();
+		fs.removeSync(testDirname);
+	});
+
+	it('should follow anchors', function(done) {
+		var options = {
+			urls: [ 'http://example.com/' ],
+			directory: testDirname,
+			subdirectories: null,
+			sources: [],
+			recursive: true
+		};
+
+		nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html');
+
+		// mock for anchors
+		nock('http://example.com/').get('/about.html').replyWithFile(200, mockDirname + '/about.html');
+		nock('http://example.com/').get('/link1.html').reply(200, 'content 1');
+		nock('http://example.com/').get('/link2.html').reply(200, 'content 2');
+		nock('http://example.com/').get('/link3.html').reply(200, 'content 3');
+
+		scraper.scrape(options).then(function() {
+			fs.existsSync(testDirname + '/index.html').should.be.eql(true);
+
+			// index.html anchors loaded
+			fs.existsSync(testDirname + '/about.html').should.be.eql(true);
+
+			// about.html anchors loaded
+			fs.existsSync(testDirname + '/link1.html').should.be.eql(true);
+			fs.existsSync(testDirname + '/link2.html').should.be.eql(true);
+			fs.existsSync(testDirname + '/link3.html').should.be.eql(true);
+
+			done();
+		}).catch(done);
+	});
+});
diff --git a/test/unit/scraper-test.js b/test/unit/scraper-test.js
@@ -141,6 +141,24 @@ describe('Scraper', function () {
 				done();
 			}).catch(done);
 		});
+
+		it('should extend sources if recursive flag is set', function(done) {
+			var s = new Scraper({
+				urls: { url: 'http://first-url.com' },
+				directory: testDirname,
+				sources: [
+					{ selector: 'img', attr: 'src' }
+				],
+				recursive: true
+			});
+
+			s.prepare().then(function() {
+				s.options.sources.should.have.length(2);
+				s.options.sources.should.containEql({ selector: 'img', attr: 'src' });
+				s.options.sources.should.containEql({ selector: 'a', attr: 'href' });
+				done();
+			}).catch(done);
+		});
 	});
 
 	describe('#load', function() {

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+module.exports = [`
	`2`	`+ { selector: 'a', attr: 'href' }`
	`3`	`+];`