Fix loading from <img srcset="">, solve #34

Author: s0ph1e

lib/config/defaults.js

@@ -5,6 +5,10 @@ var config = {
       selector: 'img',
       attr: 'src'
     },
+    {
+      selector: 'img',
+      attr: 'srcset'
+    },
     {
       selector: 'input',
       attr: 'src'

lib/file-handlers/html.js

@@ -1,16 +1,15 @@
 var cheerio = require('cheerio');
 var Promise = require('bluebird');
+var srcset = require('srcset');
 var utils = require('../utils');
 
 function loadHtml (context, resource) {
-	var sources = context.getHtmlSources();
-	var handleResources = loadResources.bind(null, context, resource);
-
+	var rules = context.getHtmlSources();
 	var p = beforeHandle(resource);
 
-	sources.forEach(function (src) {
-		p = p.then(function loadSource () {
-			return handleResources(src);
+	rules.forEach(function (rule) {
+		p = p.then(function loadResources () {
+			return loadResourcesForRule(context, resource, rule);
 		});
 	});
 	return p;
@@ -37,31 +36,91 @@ function beforeHandle (resource) {
 	return Promise.resolve(resource);
 }
 
-function loadResources (context, resource, source) {
-	var url = resource.getUrl();
-	var text = resource.getText();
-	var filename = resource.getFilename();
-	var $ = cheerio.load(text);
+/**
+ * @param {HtmlData} htmlData
+ * @returns {Function} - function which loads resources with given html data
+ */
+function getResourceLoaderByHtmlData (htmlData) {
+	if (htmlData.tagName === 'img' && htmlData.attributeName === 'srcset') {
+		return loadImgSrcsetResource;
+	}
+	return loadGeneralResource;
+}
 
-	var promises = $(source.selector).map(function loadForSelector () {
-		var el = $(this);
-		var attr = el.attr(source.attr);
+/**
+ * @param {Object} el - cheerio element
+ * @param {string} attrName - attribute name
+ * @returns {HtmlData}
+ */
+function createHtmlData (el, attrName) {
+	return {
+		tagName: el[0].name,
+		attributeName: attrName,
+		attributeValue: el.attr(attrName)
+	}
+}
+
+/**
+ * Download resources from <img srcset="...">
+ * @param context
+ * @param {Resource} parentResource
+ * @param {HtmlData} childResourceHtmlData
+ * @returns {Promise}
+ */
+function loadImgSrcsetResource (context, parentResource, childResourceHtmlData) {
+	var imgScrsetParts = srcset.parse(childResourceHtmlData.attributeValue);
+
+	return Promise.mapSeries(imgScrsetParts, function loadImgSrcsetPart (imgScrsetPart) {
+		var childResourceUrl = utils.getUrl(parentResource.getUrl(), imgScrsetPart.url);
+		var childResource = parentResource.createChild(childResourceUrl);
+		childResource.setHtmlData(childResourceHtmlData);
+
+		return context.loadResource(childResource).then(function updateSrcsetPart (loadedResource) {
+			imgScrsetPart.url = loadedResource.getFilename();
+		});
+	}).then(function updateSrcset () {
+		return Promise.resolve(srcset.stringify(imgScrsetParts));
+	});
+}
+
+/**
+ * Download common resource
+ * @param context
+ * @param {Resource} parentResource
+ * @param {HtmlData} childResourceHtmlData
+ * @returns {Promise}
+ */
+function loadGeneralResource (context, parentResource, childResourceHtmlData) {
+	var attr = childResourceHtmlData.attributeValue;
+
+	var resourceUrl = utils.getUrl(parentResource.getUrl(), attr);
+	var htmlResource = parentResource.createChild(resourceUrl);
+	htmlResource.setHtmlData(childResourceHtmlData);
 
-		if (attr) {
-			var resourceUrl = utils.getUrl(url, attr);
-			var htmlResource = resource.createChild(resourceUrl);
-			htmlResource.setHtmlData({ tagName: el[0].name, attributeName: source.attr });
+	return context.loadResource(htmlResource).then(function handleLoadedSource (loadedResource) {
+		var relativePath = utils.getRelativePath(parentResource.getFilename(), loadedResource.getFilename());
+		var hash = utils.getHashFromUrl(attr);
 
-			return context.loadResource(htmlResource).then(function handleLoadedSource (loadedResource) {
-				var relativePath = utils.getRelativePath(filename, loadedResource.getFilename());
-				var hash = utils.getHashFromUrl(attr);
+		if (hash && loadedResource.isHtml()) {
+			relativePath = relativePath.concat(hash);
+		}
 
-				if (hash && loadedResource.isHtml()) {
-					relativePath = relativePath.concat(hash);
-				}
+		return Promise.resolve(relativePath);
+	});
+}
+
+function loadResourcesForRule (context, resource, rule) {
+	var text = resource.getText();
+	var $ = cheerio.load(text);
+
+	var promises = $(rule.selector).map(function loadForElement () {
+		var el = $(this);
+		if (el.attr(rule.attr)) {
+			var childResourceHtmlData = createHtmlData(el, rule.attr);
+			var loadResourcesForElement = getResourceLoaderByHtmlData(childResourceHtmlData);
 
-				el.attr(source.attr, relativePath);
-				return Promise.resolve();
+			return loadResourcesForElement(context, resource, childResourceHtmlData).then(function changeAttr (updatedAttr) {
+				el.attr(rule.attr, updatedAttr);
 			});
 		}
 		return Promise.reject();

lib/resource.js

@@ -63,13 +63,27 @@ Resource.prototype.setDepth = function setDepth (depth) {
 };
 
 /**
+ * Html Data for resource, represents html element where resource was found
  *
- * @param {Object} data - html element data
- * @param {string} data.tagName - tag name which contain resource
- * @param {string} data.attributeName - attribute name with value of resource's url
+ * @typedef {Object} HtmlData
+ * @property {string} tagName - tag of element
+ * @property {string} attributeName - attribute in tag where resource was found
+ * @property {string} attributeValue - attribute value, contains url of resources
+ *
+ * Example: for resource in <img src="/images/foo.png"> it will be
+ *  {
+ *   tagName: 'img',
+ *   attributeName: 'src',
+ *   attributeValue: '/images/foo.png'
+ *  }
+ */
+
+/**
+ *
+ * @param {HtmlData} data
  */
 Resource.prototype.setHtmlData = function setHtmlData (data) {
-	this.htmlData = data;
+	this.htmlData = _.pick(data, ['tagName', 'attributeName']);
 };
 
 Resource.prototype.getType = function getType () {

package.json

@@ -37,6 +37,7 @@
     "css-url-parser": "^0.1.0",
     "fs-extra": "^0.26.0",
     "request": "^2.42.0",
+    "srcset": "^1.0.0",
     "underscore": "^1.7.0"
   },
   "devDependencies": {

test/unit/file-handlers/html-test.js

@@ -13,6 +13,7 @@ var defaultScraperOpts = {
 	directory: testDirname,
 	sources: [
 		{ selector: 'img', attr: 'src' },
+		{ selector: 'img', attr: 'srcset' },
 		{ selector: 'link[rel="stylesheet"]', attr: 'href' },
 		{ selector: 'script', attr: 'src'},
 		{ selector: 'a', attr: 'href' }
@@ -266,5 +267,38 @@ describe('Html handler', function () {
 			}).catch(done);
 		});
 
+		it('should handle img tag with srcset attribute correctly', function (done) {
+
+			var image45Stub = new Resource('http://example.com/image45.jpg', 'local/image45.jpg');
+			var image150Stub = new Resource('http://example.com/image150.jpg', 'local/image150.jpg');
+
+			sinon.stub(scraper, 'loadResource')
+				.onFirstCall().returns(Promise.resolve(image45Stub))
+				.onSecondCall().returns(Promise.resolve(image150Stub))
+				.onThirdCall().returns(Promise.resolve(image45Stub));
+
+			var html = '\
+				<html> \
+				<body> \
+					<img src="http://example.com/image45.jpg" \
+					srcset="http://example.com/image150.jpg 150w, http://example.com/image45.jpg 45w" \
+					sizes="(max-width: 45px) 100vw, 45px" width="45" height="45"> \
+				</body> \
+				</html>\
+			';
+
+			var po = new Resource('http://example.com', 'index.html');
+			po.setText(html);
+
+			return loadHtml(scraper, po).then(function(){
+				var text = po.getText();
+				text.should.not.containEql('http://example.com/image45.jpg');
+				text.should.not.containEql('http://example.com/image150.jpg');
+				text.should.containEql('src="local/image45.jpg"');
+				text.should.containEql('srcset="local/image150.jpg 150w, local/image45.jpg 45w"');
+				done();
+			}).catch(done);
+		});
+
 	});
 });