Update missing sources (#217)

Author: s0ph1e

README.md

@@ -62,6 +62,7 @@ scrape(options, (error, result) => {
 * [resourceSaver](#resourcesaver) - customize resources saving
 * [onResourceSaved](#onresourcesaved) - callback called when resource is saved
 * [onResourceError](#onresourceerror) - callback called when resource's downloading is failed
+* [updateMissingSources](#updatemissingsources) - update url for missing sources with absolute url
 
 Default options you can find in [lib/config/defaults.js](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/config/defaults.js) or get them using `scrape.defaults`.
 
@@ -145,7 +146,7 @@ String, filename for index page. Defaults to `index.html`.
 Boolean, whether urls should be 'prettified', by having the `defaultFilename` removed. Defaults to `false`.
 
 #### ignoreErrors
-Boolean, if `true` scraper will continue downloading resources after error occured, if `false` - scraper will finish process and return error. Defaults to `true`.
+Boolean, if `true` scraper will continue downloading resources after error occurred, if `false` - scraper will finish process and return error. Defaults to `true`.
 
 #### urlFilter
 Function which is called for each url to check whether it should be scraped. Defaults to `null` - no url filter will be applied.
@@ -253,6 +254,30 @@ scrape({
 })
 ```
 
+#### updateMissingSources
+Boolean, if `true` scraper will set absolute urls for all failing `sources`, if `false` - it will leave them as is (which may cause incorrectly displayed page).
+Also can contain array of `sources` to update (structure is similar to [sources](#sources)).
+Defaults to `false`.
+```javascript
+// update all failing img srcs with absolute url
+scrape({
+  urls: ['http://example.com/'],
+  directory: '/path/to/save',
+  sources: [{selector: 'img', attr: 'src'}],
+  updateMissingSources: true
+});
+
+// download nothing, just update all img srcs with absolute urls
+scrape({
+  urls: ['http://example.com/'],
+  directory: '/path/to/save',
+  sources: [],
+  updateMissingSources: [{selector: 'img', attr: 'src'}] 
+});
+
+```
+
+
 ## callback 
 Callback function, optional, includes following parameters:
   - `error`: if error - `Error` object, if success - `null`

lib/config/defaults.js

@@ -56,7 +56,8 @@ const config = {
 	httpResponseHandler: null,
 	onResourceSaved: null,
 	onResourceError: null,
-	resourceSaver: null
+	resourceSaver: null,
+	updateMissingSources: false
 };
 
 module.exports = config;

lib/resource-handler/css/index.js

@@ -1,16 +1,21 @@
-var CssText = require('./../path-containers/css-text');
+'use strict';
 
-function CssResourceHandler (options, handleChildrenPaths) {
-	this.options = options;
-	this.handleChildrenPaths = handleChildrenPaths;
-}
+const CssText = require('./../path-containers/css-text');
+
+class CssResourceHandler {
+	constructor (options, methods) {
+		this.options = options;
+		this.downloadChildrenPaths = methods.downloadChildrenPaths;
+		this.updateMissingSources = this.options.updateMissingSources === true || Array.isArray(this.options.updateMissingSources);
+	}
 
-CssResourceHandler.prototype.handle = function handle (resource) {
-	var pathContainer = new CssText(resource.getText());
-	return this.handleChildrenPaths(pathContainer, resource).then(function updateText (updatedText) {
-		resource.setText(updatedText);
-		return resource;
-	});
-};
+	handle (resource) {
+		const pathContainer = new CssText(resource.getText());
+		return this.downloadChildrenPaths(pathContainer, resource, this.updateMissingSources).then(function updateText (updatedText) {
+			resource.setText(updatedText);
+			return resource;
+		});
+	}
+}
 
 module.exports = CssResourceHandler;

lib/resource-handler/html/index.js

@@ -7,17 +7,30 @@ const logger = require('../../logger');
 const HtmlSourceElement = require('./html-source-element');
 
 class HtmlResourceHandler {
-	constructor (options, handleChildrenPaths) {
+	constructor (options, methods) {
 		this.options = options;
-		this.handleChildrenPaths = handleChildrenPaths;
+		this.downloadChildrenPaths = methods.downloadChildrenPaths;
+		this.updateChildrenPaths = methods.updateChildrenPaths;
+
+		this.recursiveSources = this.options.recursiveSources || [];
+		this.downloadSources = this.options.sources;
+		this.updateSources = [];
+
+		if (this.options.updateMissingSources === true) {
+			this.updateSources = this.downloadSources;
+		} else if (Array.isArray(this.options.updateMissingSources)) {
+			this.updateSources = this.options.updateMissingSources;
+		}
+
+		this.allSources = utils.union(this.downloadSources, this.updateSources);
 	}
 
 	handle (resource) {
 		const $ = loadTextToCheerio(resource.getText());
 		prepareToLoad($, resource);
 
-		return Promise.mapSeries(this.options.sources, this.loadResourcesForRule.bind(this, $, resource))
-			.then(function updateResource () {
+		return Promise.mapSeries(this.allSources, this.loadResourcesForRule.bind(this, $, resource))
+			.then(() => {
 				resource.setText($.html());
 				return resource;
 			});
@@ -27,31 +40,53 @@ class HtmlResourceHandler {
 		const self = this;
 		const promises = $(rule.selector).map((i, element) => {
 			const el = new HtmlSourceElement($(element), rule);
+			const pathContainer = el.getPathContainer();
 
-			const isRecursive = self.options.recursiveSources && Boolean(el.findMatchedRule(self.options.recursiveSources));
-			const isDepthGreaterThanMax = self.options.maxRecursiveDepth && parentResource.getDepth() >= self.options.maxRecursiveDepth;
-			if (isRecursive && isDepthGreaterThanMax) {
+			if (!pathContainer) {
+				return Promise.resolve(null);
+			}
+
+			const needToDownloadElement = this.needToDownload(el);
+			const needToUpdateElement = this.needToUpdate(el);
+
+			if (this.exceedMaxRecursiveDepth(el, parentResource)) {
 				logger.debug(`filtering out ${el} by max recursive depth`);
-				return Promise.resolve();
+				return self.updateChildrenPaths(pathContainer, parentResource, needToUpdateElement).then(el.setData.bind(el));
 			}
 
-			const pathContainer = el.getPathContainer();
-			if (!pathContainer) {
-				return Promise.resolve();
+			if (!needToDownloadElement) {
+				return self.updateChildrenPaths(pathContainer, parentResource, needToUpdateElement).then(el.setData.bind(el));
 			}
-			return self.handleChildrenPaths(pathContainer, parentResource).then((updatedText) => {
-				el.setData(updatedText);
-				el.removeIntegrityCheck();
-			});
+
+			return self.downloadChildrenPaths(pathContainer, parentResource, needToUpdateElement)
+				.then((updatedText) => {
+					el.setData(updatedText);
+					el.removeIntegrityCheck();
+				});
+
 		}).get();
 
 		return utils.waitAllFulfilled(promises);
 	}
+
+	exceedMaxRecursiveDepth (el, parentResource) {
+		const isRecursive = Boolean(el.findMatchedRule(this.recursiveSources));
+		const isDepthGreaterThanMax = this.options.maxRecursiveDepth && parentResource.getDepth() >= this.options.maxRecursiveDepth;
+		return isRecursive && isDepthGreaterThanMax;
+	}
+
+	needToDownload (el) {
+		return Boolean(el.findMatchedRule(this.downloadSources));
+	}
+
+	needToUpdate (el) {
+		return Boolean(el.findMatchedRule(this.updateSources));
+	}
 }
 
 function prepareToLoad ($, resource) {
-	$('base').each(function handleBaseTag () {
-		const el = $(this);
+	$('base').each((i, element) => {
+		const el = $(element);
 		const href = el.attr('href');
 		if (href) {
 			const newUrl = utils.getUrl(resource.getUrl(), href);

lib/resource-handler/index.js

@@ -1,82 +1,109 @@
-var _ = require('lodash');
-var Promise = require('bluebird');
-var logger = require('../logger');
-var utils = require('../utils');
+'use strict';
 
-var HtmlHandler = require('./html');
-var CssHandler = require('./css');
+const _ = require('lodash');
+const Promise = require('bluebird');
+const logger = require('../logger');
+const utils = require('../utils');
 
-var supportedOptions = ['prettifyUrls', 'sources', 'recursiveSources', 'maxRecursiveDepth', 'defaultFilename'];
+const HtmlHandler = require('./html');
+const CssHandler = require('./css');
 
-function ResourceHandler (options, context) {
-	this.options = _.pick(options, supportedOptions);
-	this.context = context;
+const supportedOptions = ['prettifyUrls', 'sources', 'recursiveSources', 'maxRecursiveDepth', 'defaultFilename', 'updateMissingSources'];
 
-	this.htmlHandler = new HtmlHandler(this.options, this.handleChildrenResources.bind(this));
-	this.cssHandler = new CssHandler(this.options, this.handleChildrenResources.bind(this));
-}
+class ResourceHandler {
+	constructor (options, context) {
+		this.options = _.pick(options, supportedOptions);
+		this.context = context;
+
+		const methods = {
+			downloadChildrenPaths: this.downloadChildrenResources.bind(this),
+			updateChildrenPaths: this.updateChildrenResources.bind(this)
+		};
 
-ResourceHandler.prototype.getResourceHandler = function getResourceHandler (resource) {
-	switch (true) {
-		case resource.isCss():
-			logger.debug('using css handler for ' + resource);
-			return this.cssHandler;
-		case resource.isHtml():
-			logger.debug('using html handler for ' + resource);
-			return this.htmlHandler;
-		default:
-			logger.debug('using no handler for ' + resource);
-			return null;
+		this.htmlHandler = new HtmlHandler(this.options, methods);
+		this.cssHandler = new CssHandler(this.options, methods);
 	}
-};
-
-/**
- * Request all resources from pathContainers paths
- * @param pathContainer - instance of ImgSrcsetTag or CommonTag or CssText, contains original paths for resources
- * @param {Resource} parentResource
- * @returns {Promise} - resolved when all resources from pathContainer were requested
- * and original paths in parentResource were updated with local paths for children resources
- */
-ResourceHandler.prototype.handleChildrenResources = function handleChildrenResources (pathContainer, parentResource) {
-	var self = this;
-	var childrenPaths = pathContainer.getPaths();
-	var pathsToUpdate = [];
-
-	var childrenPromises = childrenPaths.map(function loadChildPath (childPath) {
-		var childResourceUrl = utils.getUrl(parentResource.getUrl(), childPath);
-		var childResource = parentResource.createChild(childResourceUrl);
-
-		return self.context.requestResource(childResource).then(function updateChildPath (respondedResource) {
-			if (respondedResource) {
-				parentResource.updateChild(childResource, respondedResource);
-
-				var relativePath = utils.getRelativePath(parentResource.getFilename(), respondedResource.getFilename());
-				if (self.options.prettifyUrls) {
-					relativePath = relativePath.replace(self.options.defaultFilename, '');
-				}
-				var hash = utils.getHashFromUrl(childPath);
 
-				if (hash) {
-					relativePath = relativePath.concat(hash);
+	getResourceHandler (resource) {
+		switch (true) {
+			case resource.isCss():
+				logger.debug('using css handler for ' + resource);
+				return this.cssHandler;
+			case resource.isHtml():
+				logger.debug('using html handler for ' + resource);
+				return this.htmlHandler;
+			default:
+				logger.debug('using no handler for ' + resource);
+				return null;
+		}
+	}
+
+	/**
+	 * Request all resources from pathContainers paths
+	 * @param pathContainer - instance of ImgSrcsetTag or CommonTag or CssText, contains original paths for resources
+	 * @param {Resource} parentResource
+	 * @param {boolean} updateIfFailed - if true - failed resources will be updated with absolute links
+	 * @returns {Promise} - resolved when all resources from pathContainer were requested
+	 * and original paths in parentResource were updated with local paths for children resources
+	 */
+	downloadChildrenResources (pathContainer, parentResource, updateIfFailed) {
+		const self = this;
+		const childrenPaths = pathContainer.getPaths();
+		const pathsToUpdate = [];
+
+		const childrenPromises = childrenPaths.map((childPath) => {
+			const childResourceUrl = utils.getUrl(parentResource.getUrl(), childPath);
+			const childResource = parentResource.createChild(childResourceUrl);
+
+			return self.context.requestResource(childResource).then((respondedResource) => {
+				if (respondedResource) {
+					parentResource.updateChild(childResource, respondedResource);
+
+					let relativePath = utils.getRelativePath(parentResource.getFilename(), respondedResource.getFilename());
+					if (self.options.prettifyUrls) {
+						relativePath = relativePath.replace(self.options.defaultFilename, '');
+					}
+					const hash = utils.getHashFromUrl(childPath);
+
+					if (hash) {
+						relativePath = relativePath.concat(hash);
+					}
+
+					pathsToUpdate.push({ oldPath: childPath, newPath: relativePath});
+				} else {
+					if (updateIfFailed) {
+						pathsToUpdate.push({ oldPath: childPath, newPath: childResourceUrl});
+					}
 				}
+				return null; // Prevent Bluebird warnings
+			});
+		});
 
-				pathsToUpdate.push({ oldPath: childPath, newPath: relativePath});
-			}
-			return null; // Prevent Bluebird warnings
+		return utils.waitAllFulfilled(childrenPromises).then(function updateChildrenPaths () {
+			return pathContainer.updateText(pathsToUpdate);
 		});
-	});
+	}
 
-	return utils.waitAllFulfilled(childrenPromises).then(function updateChildrenPaths () {
-		return pathContainer.updateText(pathsToUpdate);
-	});
-};
+	updateChildrenResources (pathContainer, parentResource, needToUpdate) {
+		if (!needToUpdate) {
+			return Promise.resolve(pathContainer.updateText([]));
+		}
+		const parentUrl = parentResource.getUrl();
+		const pathsToUpdate = [];
+		pathContainer.getPaths().forEach((path) => {
+			const childAbsoluteUrl = utils.getUrl(parentUrl, path);
+			pathsToUpdate.push({ oldPath: path, newPath: childAbsoluteUrl });
+		});
+		return Promise.resolve(pathContainer.updateText(pathsToUpdate));
+	}
 
-ResourceHandler.prototype.handleResource = function handleResource (resource) {
-	var resourceHandler = this.getResourceHandler(resource);
-	if (resourceHandler && resourceHandler.handle) {
-		return resourceHandler.handle(resource);
+	handleResource (resource) {
+		const resourceHandler = this.getResourceHandler(resource);
+		if (resourceHandler && resourceHandler.handle) {
+			return resourceHandler.handle(resource);
+		}
+		return Promise.resolve(resource);
 	}
-	return Promise.resolve(resource);
-};
+}
 
 module.exports = ResourceHandler;