Add referer and decode html entities (#171)

s0ph1e · web-flow · commit a2b5c56c0607 · 2017-01-16T15:38:27.000+02:00
* Decode html entities in html resources

* Add referer to request

* Add unit tests

* Add functional test for decoding html entities

* Add missing newlines
diff --git a/lib/request.js b/lib/request.js
@@ -2,16 +2,25 @@ var _ = require('lodash');
 var Promise = require('bluebird');
 var request = require('request');
 var get = Promise.promisify(request.get);
+var logger = require('./logger');
 
 function getMimeType (contentType) {
 	return contentType ? contentType.split(';')[0] : null;
 }
 
-function makeRequest (options, url) {
+function makeRequest (options, url, referer) {
 	var requestOptions = _.clone(options);
 	requestOptions.url = url;
 
+	if (referer) {
+		requestOptions.headers = requestOptions.headers || {};
+		requestOptions.headers.referer = referer;
+	}
+
+	logger.debug(`[request] sending request for url ${url}, referer ${referer}`);
+
 	return get(requestOptions).then(function handleResponse (data) {
+		logger.debug(`[request] received response for ${data.request.href}, statusCode ${data.statusCode}`);
 		return {
 			url: data.request.href,
 			mimeType: getMimeType(data.headers['content-type']),
diff --git a/lib/resource-handler/html/html-source-element.js b/lib/resource-handler/html/html-source-element.js
@@ -2,6 +2,7 @@ var ImgSrcsetTag = require('../path-containers/html-img-srcset-tag');
 var CommonTag = require('../path-containers/html-common-tag');
 var CssText = require('../path-containers/css-text');
 var _ = require('lodash');
+var utils = require('../../utils');
 
 var pathContainersByRule = [
 	{ selector: '[style]', attr: 'style', containerClass: CssText },
@@ -22,8 +23,13 @@ function HtmlSourceElement (el, rule) {
 	this.rule = rule;
 }
 
+/**
+ * Get text from attr or from innerHtml of element based on rule
+ * @returns {string}
+ */
 HtmlSourceElement.prototype.getData = function getData () {
-	return this.rule.attr ? this.el.attr(this.rule.attr) : this.el.text();
+	var text = this.rule.attr ? this.el.attr(this.rule.attr) : this.el.text();
+	return utils.decodeHtmlEntities(text);
 };
 
 HtmlSourceElement.prototype.setData = function setData (newData) {
diff --git a/lib/scraper.js b/lib/scraper.js
@@ -111,13 +111,12 @@ Scraper.prototype.requestResource = function requestResource (resource) {
 
 	respondedResourcePromise = Promise.resolve()
 		.then(function makeRequest () {
-			logger.debug('requesting ' + url);
-			return self.makeRequest(url);
+			var referer = resource.parent ? resource.parent.getUrl() : null;
+			return self.makeRequest(url, referer);
 		}).then(function requestCompleted (responseData) {
-			logger.debug('received response for ' + url);
 
 			if (!utils.urlsEqual(responseData.url, url)) { // Url may be changed in redirects
-				logger.debug('url changed. old url = ' + url + ', new ulr = ' + responseData.url);
+				logger.debug('url changed. old url = ' + url + ', new url = ' + responseData.url);
 				var respondedNewUrlResource = self.getRespondedResourcePromise(responseData.url);
 				if (respondedNewUrlResource) {
 					return respondedNewUrlResource;
@@ -138,8 +137,6 @@ Scraper.prototype.requestResource = function requestResource (resource) {
 			}
 
 			resource.setText(responseData.body);
-
-			logger.debug('finish request for ' + resource);
 			return resource;
 		}).catch(function handleError (err) {
 			logger.warn('failed to request resource ' + resource);
diff --git a/lib/utils.js b/lib/utils.js
@@ -2,6 +2,7 @@ var url = require('url');
 var path = require('path');
 var Promise = require('bluebird');
 var normalizeUrl = require('normalize-url');
+var htmlEntities = require('he');
 var typeByMime = require('./config/resource-type-by-mime');
 var typeByExt = require('./config/resource-type-by-ext');
 
@@ -102,6 +103,10 @@ function getTypeByFilename (filename) {
 	return typeByExt[ext];
 }
 
+function decodeHtmlEntities (text) {
+	return typeof text === 'string' ? htmlEntities.decode(text) : '';
+}
+
 module.exports = {
 	isUrl: isUrl,
 	getUrl: getUrl,
@@ -117,5 +122,6 @@ module.exports = {
 	urlsEqual: urlsEqual,
 	isUriSchemaSupported: isUriSchemaSupported,
 	getTypeByMime: getTypeByMime,
-	getTypeByFilename: getTypeByFilename
+	getTypeByFilename: getTypeByFilename,
+	decodeHtmlEntities: decodeHtmlEntities
 };
diff --git a/package.json b/package.json
@@ -38,6 +38,7 @@
     "css-url-parser": "^1.0.0",
     "debug": "^2.4.5",
     "fs-extra": "^1.0.0",
+    "he": "^1.1.0",
     "lodash": "^4.11.1",
     "normalize-url": "^1.5.3",
     "request": "^2.42.0",
diff --git a/test/functional/html-entities-in-url/html-entities-in-url.test.js b/test/functional/html-entities-in-url/html-entities-in-url.test.js
@@ -0,0 +1,80 @@
+var should = require('should');
+var nock = require('nock');
+var fs = require('fs-extra');
+var Scraper = require('../../../lib/scraper');
+
+var testDirname = __dirname + '/.tmp';
+var mockDirname = __dirname + '/mocks';
+
+describe('Functional: html entities in url', function() {
+
+	beforeEach(function() {
+		nock.cleanAll();
+		nock.disableNetConnect();
+	});
+
+	afterEach(function() {
+		nock.cleanAll();
+		nock.enableNetConnect();
+		fs.removeSync(testDirname);
+	});
+
+	it('should decode all html-entities found in html files and not encode entities from css file', function() {
+		nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html');
+		nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css');
+
+		// in index.html
+		// /fonts?family=Myriad&amp;v=2 => /fonts?family=Myriad&v=2
+		nock('http://example.com/').get('/fonts?family=Myriad&v=2').reply(200, 'fonts.css', {'content-type': 'text/css'});
+		// /?a=1&amp;style-attr.png => /?a=1&style-attr.png
+		nock('http://example.com/').get('/style-attr.png?a=1&style-attr.png').reply(200, 'style-attr.png', {'content-type': 'text/css'});
+		// /?a=1&amp;b=2 => /?a=1&b=2
+		nock('http://example.com/').get('/img.png?a=1&b=2').reply(200, 'img.png');
+		// /test?b=2&amp;c=3&amp;d=4 => /test?b=2&c=3&d=4
+		nock('http://example.com/').get('/?b=2&c=3&d=4').reply(200, 'index_1.html', {'content-type': 'text/html'});
+
+		// in style.css
+		// /?v=2&amp;name=external-style.png should stay not decoded
+		nock('http://example.com/').get('/external-style.png?v=2&amp;name=external-style.png').reply(200, 'external-style.png');
+
+		var options = {
+			urls: [ 'http://example.com/' ],
+			directory: testDirname,
+			maxDepth: 2,
+			recursive: true,
+			subdirectories: [
+				{ directory: 'local', extensions: ['.png', '.css'] }
+			],
+			ignoreErrors: false
+		};
+		var scraper = new Scraper(options);
+
+		return scraper.scrape(options).then(function() {
+			fs.existsSync(testDirname + '/index.html').should.be.eql(true);
+			var indexHtml = fs.readFileSync(testDirname + '/index.html').toString();
+
+			should(indexHtml).containEql('href="local/fonts.css');
+			fs.existsSync(testDirname + '/local/fonts.css').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/local/fonts.css').toString()).be.eql('fonts.css');
+
+			should(indexHtml).containEql('background: url(\'local/style-attr.png\')');
+			fs.existsSync(testDirname + '/local/style-attr.png').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/local/style-attr.png').toString()).be.eql('style-attr.png');
+
+			should(indexHtml).containEql('img src="local/img.png');
+			fs.existsSync(testDirname + '/local/img.png').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/local/img.png').toString()).be.eql('img.png');
+
+			should(indexHtml).containEql('href="index_1.html"');
+			fs.existsSync(testDirname + '/index_1.html').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/index_1.html').toString()).be.eql('index_1.html');
+
+			fs.existsSync(testDirname + '/local/style.css').should.be.eql(true);
+			var styleCss = fs.readFileSync(testDirname + '/local/style.css').toString();
+
+			should(styleCss).containEql('url(\'external-style.png\')');
+			fs.existsSync(testDirname + '/local/external-style.png').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/local/external-style.png').toString()).be.eql('external-style.png');
+		});
+	});
+});
diff --git a/test/functional/html-entities-in-url/mocks/index.html b/test/functional/html-entities-in-url/mocks/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <link rel="stylesheet" type="text/css" href="style.css"/>
+    <link rel="stylesheet" href="http://example.com/fonts?family=Myriad&amp;v=2" type="text/css" media="all" />
+</head>
+<body>
+    <div style="background: url('http://example.com/style-attr.png?a=1&amp;style-attr.png')"></div>
+    <img src="http://example.com/img.png?a=1&amp;b=2" />
+    <a href="?b=2&amp;c=3&amp;d=4">test</a>
+</body>
+</html>
diff --git a/test/functional/html-entities-in-url/mocks/style.css b/test/functional/html-entities-in-url/mocks/style.css
@@ -0,0 +1,3 @@
+.bg {
+    background-image: url('http://example.com/external-style.png?v=2&amp;name=external-style.png');
+}
diff --git a/test/unit/filename-generator/by-site-structure-test.js b/test/unit/filename-generator/by-site-structure-test.js
@@ -7,7 +7,7 @@ var bySiteStructureFilenameGenerator = require('../../../lib/filename-generator/
 
 var options = { defaultFilename: 'index.html' };
 
-describe('byStructureFilenameGenerator', function() {
+describe('FilenameGenerator: bySiteStructure', function() {
 	it('should return the normalized relative path of the resource url', function(){
 		var r1 = new Resource('http://example.com/some/path/a.png');
 		bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('some/path/a.png');
diff --git a/test/unit/filename-generator/by-type-test.js b/test/unit/filename-generator/by-type-test.js
@@ -5,7 +5,7 @@ var sinon = require('sinon');
 var Resource = require('../../../lib/resource');
 var byTypeFilenameGenerator = require('../../../lib/filename-generator/by-type');
 
-describe('byTypeFilenameGenerator', function() {
+describe('FilenameGenerator: byType', function() {
 	it('should return resource filename', function() {
 		var r = new Resource('http://example.com/a.png', 'b.png');
 		var filename = byTypeFilenameGenerator(r, {}, []);
diff --git a/test/unit/request-test.js b/test/unit/request-test.js
@@ -1,66 +1,93 @@
-require('should');
-
+var should = require('should');
 var nock = require('nock');
 var sinon = require('sinon');
 require('sinon-as-promised');
 var proxyquire = require('proxyquire');
-var request = require('../../lib/request');
 
 describe('Request', function () {
+	var makeRequest, makeStubbedRequest, requestStub;
 
 	beforeEach(function() {
 		nock.cleanAll();
 		nock.enableNetConnect();
+
+		makeRequest = require('../../lib/request');
+
+		requestStub = sinon.stub().yields(null, { request: {href: ''}, body: '', headers: {} });
+		makeStubbedRequest = proxyquire('../../lib/request', {
+			'request': {
+				'get': requestStub
+			}
+		});
 	});
 
 	afterEach(function() {
 		nock.cleanAll();
 		nock.enableNetConnect();
 	});
 
-	describe('#makeRequest', function () {
+	it('should call request with correct params', function () {
+		var options = {
+			headers: {
+				'User-Agent': 'Mozilla/5.0 (Linux; Android 4.2.1;'
+			}
+		};
+		var url = 'http://www.google.com';
 
-		it('should call request with correct params', function(done) {
-			var responseMock = { request: {href: ''}, body: '', headers: {} };
-			var requestStub = sinon.stub().yields(null, responseMock);
-
-			var customRequest = proxyquire('../../lib/request', {
-				'request': {
-					'get': requestStub
-				}
-			});
-			var options = {
+		return makeStubbedRequest(options, url).then(function () {
+			var expectedOptions = {
 				headers: {
 					'User-Agent': 'Mozilla/5.0 (Linux; Android 4.2.1;'
-				}
+				},
+				url: url
+			};
+
+			requestStub.calledOnce.should.be.eql(true);
+			requestStub.calledWith(expectedOptions).should.be.eql(true);
+		});
+	});
+
+	it('should add referer header if referer param was passed', function() {
+		var options = {};
+		var url = 'http://www.google.com';
+		var referer = 'http://referer.com';
+
+		return makeStubbedRequest(options, url, referer).then(function () {
+			var expectedOptions = {
+				headers: {
+					referer: referer
+				},
+				url: url
 			};
-			var url = 'http://www.google.com';
 
-			customRequest(options, url).then(function () {
-				var expectedOptions = {
-					headers: {
-						'User-Agent': 'Mozilla/5.0 (Linux; Android 4.2.1;'
-					},
-					url: url
-				};
+			requestStub.calledOnce.should.be.eql(true);
+			requestStub.calledWith(expectedOptions).should.be.eql(true);
+		});
+	});
 
-				requestStub.calledOnce.should.be.eql(true);
-				requestStub.calledWith(expectedOptions).should.be.eql(true);
-				done();
-			}).catch(done);
+	it('should return object with url, body and mimeType properties', function () {
+		var url = 'http://www.google.com';
+		nock(url).get('/').reply(200, 'Hello from Google!', {
+			'content-type': 'text/html; charset=utf-8'
+		});
 
+		return makeRequest({}, url).then(function (data) {
+			data.should.have.properties(['url', 'body', 'mimeType']);
+			data.url.should.be.eql('http://www.google.com/');
+			data.body.should.be.eql('Hello from Google!');
+			data.mimeType.should.be.eql('text/html');
 		});
+	});
 
-		it('should return object with url and body properties', function (done) {
-			var url = 'http://www.google.com';
-			nock(url).get('/').reply(200, 'Hello from Google!');
+	it('should return mimeType = null if content-type header was not found in response', function () {
+		var url = 'http://www.google.com';
+		nock(url).get('/').reply(200, 'Hello from Google!', {});
 
-			request({}, url).then(function (data) {
-				data.should.have.properties(['url', 'body']);
-				data.url.should.be.eql('http://www.google.com/');
-				data.body.should.be.eql('Hello from Google!');
-				done();
-			}).catch(done);
+		return makeRequest({}, url).then(function (data) {
+			data.should.have.properties(['url', 'body', 'mimeType']);
+			data.url.should.be.eql('http://www.google.com/');
+			data.body.should.be.eql('Hello from Google!');
+			should(data.mimeType).be.eql(null);
 		});
 	});
 });
diff --git a/test/unit/utils-test.js b/test/unit/utils-test.js
@@ -3,7 +3,7 @@ var _ = require('lodash');
 var utils = require('../../lib/utils');
 var Promise = require('bluebird');
 
-describe('Common utils', function () {
+describe('Utils', function () {
 	describe('#isUrl(url)', function () {
 		it('should return true if url starts with "http[s]://"', function () {
 			utils.isUrl('http://google.com').should.be.true();
@@ -197,4 +197,17 @@ describe('Common utils', function () {
 			should(utils.isUriSchemaSupported('index.html')).be.eql(true);
 		});
 	});
+
+	describe('#decodeHtmlEntities', function() {
+		it('should return empty string if not string passed', function() {
+			should(utils.decodeHtmlEntities(null)).be.eql('');
+			should(utils.decodeHtmlEntities({})).be.eql('');
+			should(utils.decodeHtmlEntities([])).be.eql('');
+			should(utils.decodeHtmlEntities()).be.eql('');
+		});
+
+		it('should return decoded text if string passed', function() {
+			should(utils.decodeHtmlEntities('?a=1&amp;v=2')).be.eql('?a=1&v=2');
+		});
+	});
 });

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+.bg {`
	`2`	`+ background-image: url('http://example.com/external-style.png?v=2&name=external-style.png');`
	`3`	`+}`