Add saved and error callbacks (#190)

Author: s0ph1e

README.md

@@ -58,6 +58,8 @@ scrape(options, (error, result) => {
 * [urlFilter](#urlfilter) - skip some urls
 * [filenameGenerator](#filenamegenerator) - generate filename for downloaded resource
 * [httpResponseHandler](#httpresponsehandler) - customize http response handling
+* [onResourceSaved](#onresourcesaved) - callback called when resource is saved
+* [onResourceError](#onresourceerror) - callback called when resource's downloading is failed
 
 Default options you can find in [lib/config/defaults.js](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/config/defaults.js).
 
@@ -209,6 +211,30 @@ scrape({
 ```
 Scrape function resolves with array of [Resource](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/resource.js) objects which contain `metadata` property from `httpResponseHandler`. 
 
+#### onResourceSaved
+Function called each time when resource is saved to file system. Callback is called with [Resource](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/resource.js) object. Defaults to `null` - no callback will be called.
+```javascript
+scrape({
+  urls: ['http://example.com/'],
+  directory: '/path/to/save',
+  onResourceSaved: (resource) => {
+  	console.log(`Resource ${resource} was saved to fs`);
+  }
+})
+```
+
+#### onResourceError
+Function called each time when resource's downloading/handling/saving to fs was failed. Callback is called with - [Resource](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/resource.js) object and `Error` object. Defaults to `null` - no callback will be called.
+```javascript
+scrape({
+  urls: ['http://example.com/'],
+  directory: '/path/to/save',
+  onResourceError: (resource, err) => {
+  	console.log(`Resource ${resource} was not saved because of ${err}`);
+  }
+})
+```
+
 ## callback 
 Callback function, optional, includes following parameters:
   - `error`: if error - `Error` object, if success - `null`

lib/config/defaults.js

@@ -34,7 +34,9 @@ var config = {
 	recursive: false,
 	maxDepth: null,
 	ignoreErrors: true,
-	httpResponseHandler: null
+	httpResponseHandler: null,
+	onResourceSaved: null,
+	onResourceError: null
 };
 
 module.exports = config;

lib/scraper.js

@@ -67,9 +67,13 @@ Scraper.prototype.saveResource = function saveResource (resource) {
 		}).then(function fileHandled () {
 			logger.info('saving resource ' + resource + ' to fs');
 			return self.fsAdapter.saveResource(resource);
+		}).then(function afterResourceSaved () {
+			if (self.options.onResourceSaved) {
+				self.options.onResourceSaved(resource);
+			}
 		}).catch(function handleError (err) {
 			logger.warn('failed to save resource ' + resource);
-			return self.handleError(err);
+			return self.handleError(err, resource);
 		});
 };
 
@@ -113,7 +117,7 @@ Scraper.prototype.createNewRequest = function createNewRequest (resource) {
 			return resource;
 		}).catch(function handleError (err) {
 			logger.warn('failed to request resource ' + resource);
-			return self.handleError(err);
+			return self.handleError(err, resource);
 		});
 
 	self.requestedResourcePromises.set(url, requestPromise);
@@ -169,7 +173,10 @@ Scraper.prototype.waitForLoad = function waitForLoad () {
 	return Promise.resolve(self.originalResources);
 };
 
-Scraper.prototype.handleError = function handleError (err) {
+Scraper.prototype.handleError = function handleError (err, resource) {
+	if (resource && this.options.onResourceError) {
+		this.options.onResourceError(resource, err);
+	}
 	if (this.options.ignoreErrors) {
 		logger.warn('ignoring error: ' + err.message);
 		return Promise.resolve(null);

test/functional/callbacks/callbacks.test.js

@@ -0,0 +1,47 @@
+const should = require('should');
+const nock = require('nock');
+const fs = require('fs-extra');
+const sinon = require('sinon');
+const scrape = require('../../../index');
+
+const testDirname = __dirname + '/.tmp';
+const mockDirname = __dirname + '/mocks';
+
+describe('Functional callbacks', () => {
+
+	beforeEach(() => {
+		nock.cleanAll();
+		nock.disableNetConnect();
+	});
+
+	afterEach(() => {
+		nock.cleanAll();
+		nock.enableNetConnect();
+		fs.removeSync(testDirname);
+	});
+
+	it('should call onResourceSaved callback and onResourceError callback', function() {
+		nock('http://example.com/').get('/').reply(200, 'OK');
+		nock('http://nodejs.org/').get('/').replyWithError('REQUEST ERROR!!');
+
+		const resourceSavedStub = sinon.stub();
+		const resourceErrorStub = sinon.stub();
+
+		const options = {
+			urls: [ 'http://example.com/', 'http://nodejs.org/' ],
+			directory: testDirname,
+			subdirectories: null,
+			onResourceSaved: resourceSavedStub,
+			onResourceError: resourceErrorStub
+		};
+
+		return scrape(options).then(function() {
+			should(resourceSavedStub.calledOnce).be.eql(true);
+			should(resourceSavedStub.args[0][0].url).be.eql('http://example.com/');
+
+			should(resourceErrorStub.calledOnce).be.eql(true);
+			should(resourceErrorStub.args[0][0].url).be.eql('http://nodejs.org/');
+			should(resourceErrorStub.args[0][1].message).be.eql('REQUEST ERROR!!');
+		});
+	});
+});