Use async functions, improve normalize method

Author: s0ph1e

lib/scraper.js

@@ -62,12 +62,13 @@ class Scraper {
 		this.options = extend(defaults, options);
 		this.options.request = extend(defaults.request, options.request);
 
-		this.options.urls = Array.isArray(this.options.urls) ? this.options.urls : [this.options.urls];
-		this.options.urls.forEach((urlItem, i) => {
+		const urls = Array.isArray(options.urls) ? options.urls : [options.urls];
+
+		this.options.urls = urls.map((urlItem) => {
 			if (typeof urlItem === 'string') {
-				this.options.urls[i] = {url: urlItem, filename: this.options.defaultFilename};
+				return { url: urlItem, filename: this.options.defaultFilename };
 			} else {
-				this.options.urls[i] = {url: urlItem.url, filename: urlItem.filename || this.options.defaultFilename};
+				return {url: urlItem.url, filename: urlItem.filename || this.options.defaultFilename};
 			}
 		});
 
@@ -125,21 +126,19 @@ class Scraper {
 		}
 	}
 
-	saveResource (resource) {
+	async saveResource (resource) {
 		resource.setSaved();
 
-		return Promise.resolve()
-			.then(() => this.resourceHandler.handleResource(resource))
-			.then(() => {
-				logger.info('saving resource ' + resource + ' to fs');
-				return this.runActions('saveResource', {resource});
-			}).then(() => {
-				// ignore promise here, just notifying external code about resource saved
-				this.runActions('onResourceSaved', {resource});
-			}).catch((err) => {
-				logger.warn('failed to save resource ' + resource);
-				return this.handleError(err, resource);
-			});
+		try {
+			await this.resourceHandler.handleResource(resource);
+			logger.info('saving resource ' + resource + ' to fs');
+			await this.runActions('saveResource', {resource});
+			// ignore promise here, just notifying external code about resource saved
+			this.runActions('onResourceSaved', {resource});
+		} catch (err) {
+			logger.warn('failed to save resource ' + resource);
+			return this.handleError(err, resource);
+		}
 	}
 
 	createNewRequest (resource) {
@@ -199,17 +198,17 @@ class Scraper {
 		return requestPromise;
 	}
 
-	requestResource (resource) {
+	async requestResource (resource) {
 		const url = resource.getUrl();
 
 		if (this.options.urlFilter && !this.options.urlFilter(url)) {
 			logger.debug('filtering out ' + resource + ' by url filter');
-			return Promise.resolve(null);
+			return null;
 		}
 
 		if (this.options.maxDepth && resource.getDepth() > this.options.maxDepth) {
 			logger.debug('filtering out ' + resource + ' by depth');
-			return Promise.resolve(null);
+			return null;
 		}
 
 		if (this.requestedResourcePromises.has(url)) {
@@ -254,15 +253,15 @@ class Scraper {
 		return Promise.resolve(this.resources);
 	}
 
-	handleError (err, resource) {
+	async handleError (err, resource) {
 		// ignore promise here, just notifying external code about resource error
 		this.runActions('onResourceError', {resource, error: err});
 
 		if (this.options.ignoreErrors) {
 			logger.warn('ignoring error: ' + err.message);
-			return Promise.resolve(null);
+			return null;
 		}
-		return Promise.reject(err);
+		throw err;
 	}
 
 	async errorCleanup (error) {