Merge pull request #67 from s0ph1e/createoutput-referential-loop-bug

Remove createOutputObject and return resources as-is

Author: Mark Lagendijk

README.md

@@ -59,10 +59,10 @@ Makes requests to `urls` and saves all files found with `sources` to `directory`
 **callback** - callback function *(optional)*, includes following parameters:
 
   - `error`: if error - `Error` object, if success - `null`
-  - `result`: if error - `null`, if success - array of objects containing:
+  - `result`: if error - `null`, if success - array of [Resource](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/resource.js) objects containing:
     - `url`: url of loaded page
     - `filename`: filename where page was saved (relative to `directory`)
-    - `assets`: array of children resources (each of them contains `url`, `filename`, `assets`)
+    - `children`: array of children Resources
 
 ### Filename Generators
 The filename generator determines where the scraped files are saved.

lib/scraper.js

@@ -10,7 +10,6 @@ var _ = require('lodash');
 
 var defaults = require('./config/defaults');
 var recursiveSources = require('./config/recursive-sources');
-var utils = require('./utils.js');
 var Resource = require('./resource');
 
 var getFilenameGenerator = require('./filename-generators/filename-generator-getter');
@@ -128,8 +127,8 @@ Scraper.prototype.prepare = function prepare () {
 
 Scraper.prototype.load = function load () {
 	var self = this;
-	return Promise.map(self.originalResources, function loadPage (po) {
-		return self.loadResource(po).then(utils.createOutputObject);
+	return Promise.map(self.originalResources, function loadResource (resource) {
+		return self.loadResource(resource);
 	});
 };
 

lib/utils.js

@@ -1,6 +1,5 @@
 var url = require('url');
 var path = require('path');
-var _ = require('lodash');
 var Promise = require('bluebird');
 
 function isUrl (path) {
@@ -42,29 +41,12 @@ function waitAllFulfilled (promises) {
 	}));
 }
 
-function createOutputObject (resource, outputObjectsByUrl) {
-	outputObjectsByUrl = outputObjectsByUrl || {};
-
-	var outputObject = {
-		url: resource.getUrl(),
-		filename: resource.getFilename()
-	};
-	outputObjectsByUrl[outputObject.url] = outputObject;
-
-	outputObject.assets = _.map(resource.getChildren(), function getOrCreateChildOutputObject (childResource){
-		return outputObjectsByUrl[childResource.getUrl()] || createOutputObject(childResource, outputObjectsByUrl);
-	});
-
-	return outputObject;
-}
-
 module.exports = {
 	isUrl: isUrl,
 	getUrl: getUrl,
 	getUnixPath: getUnixPath,
 	getRelativePath: getRelativePath,
 	getFilenameFromUrl: getFilenameFromUrl,
 	getHashFromUrl: getHashFromUrl,
-	waitAllFulfilled: waitAllFulfilled,
-	createOutputObject: createOutputObject
+	waitAllFulfilled: waitAllFulfilled
 };

test/functional/base-test.js

@@ -3,6 +3,7 @@ var nock = require('nock');
 var fs = require('fs-extra');
 var cheerio = require('cheerio');
 var scraper = require('../../index');
+var Resource = require('../../lib/resource');
 
 var testDirname = __dirname + '/.base';
 var mockDirname = __dirname + '/mocks/base';
@@ -68,16 +69,19 @@ describe('Functional base', function() {
 			result.should.be.instanceOf(Array).and.have.length(3);
 
 			result[0].should.have.properties({ url: 'http://example.com/', filename: 'index.html' });
-			result[0].should.have.properties('assets');
-			result[0].assets.should.be.instanceOf(Array).and.have.length(4);
+			result[0].should.have.properties('children');
+			result[0].children.should.be.instanceOf(Array).and.have.length(4);
+			result[0].children[0].should.be.instanceOf(Resource);
 
 			result[1].should.have.properties({ url: 'http://example.com/about', filename: 'about.html' });
-			result[1].should.have.properties('assets');
-			result[1].assets.should.be.instanceOf(Array).and.have.length(4);
+			result[1].should.have.properties('children');
+			result[1].children.should.be.instanceOf(Array).and.have.length(4);
+			result[1].children[0].should.be.instanceOf(Resource);
 
 			result[2].should.have.properties({ url: 'http://blog.example.com/', filename: 'blog.html' }); // url after redirect
-			result[2].should.have.properties('assets');
-			result[2].assets.should.be.instanceOf(Array).and.have.length(1);
+			result[2].should.have.properties('children');
+			result[2].children.should.be.instanceOf(Array).and.have.length(1);
+			result[2].children[0].should.be.instanceOf(Resource);
 
 			// should create directory and subdirectories
 			fs.existsSync(testDirname).should.be.eql(true);

test/unit/scraper-test.js

@@ -210,7 +210,7 @@ describe('Scraper', function () {
 			}).catch(done);
 		});
 
-		it('should return array of objects with url, filename and assets', function(done) {
+		it('should return array of objects with url, filename and childre', function(done) {
 			nock('http://first-url.com').get('/').reply(200, 'OK');
 			nock('http://second-url.com').get('/').reply(500);
 
@@ -225,8 +225,8 @@ describe('Scraper', function () {
 			s.prepare().bind(s).then(s.load).then(function(res) {
 				res.should.be.instanceOf(Array);
 				res.should.have.length(2);
-				res[0].should.have.properties(['url', 'filename', 'assets']);
-				res[1].should.have.properties(['url', 'filename', 'assets']);
+				res[0].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'children']);
+				res[1].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'children']);
 				done();
 			}).catch(done);
 		});

test/unit/utils-test.js

@@ -83,29 +83,4 @@ describe('Common utils', function () {
 			utils.getRelativePath('css/1.css', 'css/2.css').should.be.equal('2.css');
 		});
 	});
-
-	describe('#createOutputObject', function () {
-		it('should create output object recursively', function() {
-			var root = new Resource('http://google.com', 'google.html');
-			root.createChild('http://child-one.com', 'child.html');
-			root.createChild('http://child-two.com', 'child2.html');
-
-			var expected = {
-				url: 'http://google.com',
-				filename: 'google.html',
-				assets: [{
-					url: 'http://child-one.com',
-					filename: 'child.html',
-					assets: []
-				}, {
-					url: 'http://child-two.com',
-					filename: 'child2.html',
-					assets: []
-				}]
-			};
-
-			var result = utils.createOutputObject(root);
-			result.should.eql(expected);
-		});
-	});
 });