Improve recognizing resource type (using parent resource & html data) + test

Author: s0ph1e

lib/config/resource-types-by-tag.js

@@ -0,0 +1,13 @@
+var types = require('./resource-types');
+
+var typesByHtmlTag = {};
+
+typesByHtmlTag[types.css] = [
+	{ tagName: 'link', attributeName: 'href' }
+];
+typesByHtmlTag[types.html] = [
+	{ tagName: 'a', attributeName: 'href' },
+	{ tagName: 'iframe', attributeName: 'src' }
+];
+
+module.exports = typesByHtmlTag;

lib/config/resource-types.js

@@ -0,0 +1,7 @@
+var types = {
+	css: 'css',
+	html: 'html',
+	other: 'other'
+};
+
+module.exports = types;

lib/file-handlers/css.js

@@ -13,6 +13,7 @@ function loadCss (context, resource) {
 	var promises = _.map(cssUrls, function loadResourceFromCssUrl (cssUrl) {
 		var resourceUrl = utils.getUrl(url, cssUrl);
 		var cssResource = new Resource(resourceUrl);
+		cssResource.setParent(resource);
 
 		return context.loadResource(cssResource).then(function handleLoadedSource (loadedResource) {
 			var relativePath = utils.getRelativePath(filename, loadedResource.getFilename());

lib/file-handlers/html.js

@@ -1,6 +1,5 @@
 var Promise = require('bluebird');
 var cheerio = require('cheerio');
-var _ = require('underscore');
 var utils = require('../utils');
 var Resource = require('../resource');
 
@@ -10,7 +9,7 @@ function loadHtml (context, resource) {
 
 	var p = beforeHandle(resource);
 
-	_.each(sources, function (src) {
+	sources.forEach(function (src) {
 		p = p.then(function loadSource () {
 			return handleResources(src);
 		});
@@ -52,6 +51,8 @@ function loadResources (context, resource, source) {
 		if (attr) {
 			var resourceUrl = utils.getUrl(url, attr);
 			var htmlResource = new Resource(resourceUrl);
+			htmlResource.setParent(resource);
+			htmlResource.setHtmlData({ tagName: el[0].name, attributeName: source.attr });
 
 			return context.loadResource(htmlResource).then(function handleLoadedSource (loadedResource) {
 				var relativePath = utils.getRelativePath(filename, loadedResource.getFilename());

lib/resource.js

@@ -1,4 +1,14 @@
+var _ = require('underscore');
 var path = require('path');
+var types = require('./config/resource-types');
+var typesByHtmlData = require('./config/resource-types-by-tag');
+
+function getTypeByHtmlData (htmlData) {
+	var type = _.findKey(typesByHtmlData, function containsHtmlData (rules) {
+		return _.findWhere(rules, htmlData);
+	});
+	return type || types.other;
+}
 
 function Resource (url, filename) {
 	this.url = url;
@@ -29,15 +39,35 @@ Resource.prototype.setText = function setText (text) {
 	this.text = text;
 };
 
+Resource.prototype.setParent = function setParent (parent) {
+	this.parent = parent;
+};
+
+/**
+ *
+ * @param {Object} data - html element data
+ * @param {string} data.tagName - tag name which contain resource
+ * @param {string} data.attributeName - attribute name with value of resource's url
+ */
+Resource.prototype.setHtmlData = function setHtmlData (data) {
+	this.htmlData = data;
+};
+
 Resource.prototype.getType = function getType () {
 	var ext = path.extname(this.filename);
-	switch (ext) {
-		case '.css':
-			return 'css';
-		case '.html':
-			return 'html';
+	var parentType = this.parent && this.parent.getType();
+	var hasHtmlData = !!this.htmlData;
+
+	switch (true) {
+		case ext == '.html':
+			return types.html;
+		case ext == '.css':
+		case !ext && parentType == types.css:
+			return types.css;
+		case !ext && parentType == types.html && hasHtmlData:
+			return getTypeByHtmlData(this.htmlData);
 		default:
-			return 'other';
+			return types.other;
 	}
 };
 

lib/scraper.js

@@ -3,7 +3,8 @@ var fs = Promise.promisifyAll(require('fs-extra'));
 var path = require('path');
 var _ = require('underscore');
 
-var defaults = require('./config/defaults.js');
+var defaults = require('./config/defaults');
+var types = require('./config/resource-types');
 var utils = require('./utils.js');
 var request = require('./request');
 var Resource = require('./resource');
@@ -37,8 +38,8 @@ function generateFilename (resource) {
 	var occupiedFilenames = self.getOccupiedFilenames();
 
 	var preferedFilename = resource.getFilename();       // which was set in options
-	var urlFilename = path.basename(resource.getUrl());  // try to get filename from url
-	var filename = utils.trimFilename(preferedFilename || urlFilename);
+	var urlFilename = utils.getFilenameFromUrl(resource.getUrl());  // try to get filename from url
+	var filename = preferedFilename || urlFilename || self.options.defaultFilename;
 
 	var ext = path.extname(filename);
 	var dir = self.getDirectoryByExtension(ext);
@@ -64,8 +65,8 @@ function getDirectoryByExtension (ext) {
 function getHandleFunction (resource) {
 	var type = resource.getType();
 	switch (type) {
-		case 'css': return loadCss;
-		case 'html': return function loadHtmlAndCss (context, po) {
+		case types.css: return loadCss;
+		case types.html: return function loadHtmlAndCss (context, po) {
 			return loadHtml(context, po).then(function (loaded) {
 				return loadCss(context, loaded);
 			});

lib/utils.js

@@ -1,11 +1,6 @@
 var url = require('url');
 var path = require('path');
 
-function trimFilename(filename) {
-	var queryRegexp = /[\?#](.*)$/;
-	return filename.replace(queryRegexp, '');
-}
-
 function isUrl(path) {
 	var urlRegexp = /^((http[s]?:)?\/\/)/;
 	return urlRegexp.test(path);
@@ -30,8 +25,14 @@ function getRelativePath(path1, path2) {
 	return getUnixPath(relativePath);
 }
 
-module.exports.isUrl = isUrl;
-module.exports.getUrl = getUrl;
-module.exports.getUnixPath = getUnixPath;
-module.exports.trimFilename = trimFilename;
-module.exports.getRelativePath = getRelativePath;
+function getFilenameFromUrl (u) {
+	return path.basename(url.parse(u).pathname);
+}
+
+module.exports = {
+	isUrl: isUrl,
+	getUrl: getUrl,
+	getUnixPath: getUnixPath,
+	getRelativePath: getRelativePath,
+	getFilenameFromUrl: getFilenameFromUrl
+};

test/functional/complex-test.js renamed to test/functional/base-test.js

@@ -6,10 +6,10 @@ var _ = require('underscore');
 var cheerio = require('cheerio');
 var scraper = require('../../index');
 
-var testDirname = __dirname + '/.complex';
-var mockDirname = __dirname + '/mocks/complex';
+var testDirname = __dirname + '/.base';
+var mockDirname = __dirname + '/mocks/base';
 
-describe('Functional', function() {
+describe('Functional base', function() {
 
 	beforeEach(function() {
 		nock.cleanAll();