Fix incorrect filepath from url, fixes #140 (#142)

* Fix incorrect filepath from url, fixes #140

* Add default filenames for html resources with incorrect extension

Author: s0ph1e

lib/config/resource-extensions-by-type.js

@@ -2,7 +2,13 @@ var types = require('./resource-types');
 
 var defaultExtensions = {};
 
-defaultExtensions[types.html] = '.html';
-defaultExtensions[types.css] = '.css';
+defaultExtensions[types.html] = {
+	defaultExtension: '.html',
+	possibleExtensions: [ '.html', '.htm' ]
+};
+defaultExtensions[types.css] = {
+	defaultExtension: '.css',
+	possibleExtensions: [ '.css' ]
+};
 
 module.exports = defaultExtensions;

lib/filename-generators/by-site-structure.js

@@ -1,15 +1,22 @@
 var _ = require('lodash');
 var path = require('path');
 var utils = require('../utils');
+var resourceTypes = require('../config/resource-types');
+var resourceTypeExtensions = require('../config/resource-extensions-by-type');
 
 module.exports = function generateFilename (resource, options) {
 	var resourceUrl = resource.getUrl();
 	var filePath = utils.getFilepathFromUrl(resourceUrl);
 	var extension = utils.getFilenameExtension(filePath);
 
 	// If we have HTML from 'http://example.com/path' => set 'path/index.html' as filepath
-	if (resource.isHtml() && !extension) {
-		filePath = path.join(filePath, options.defaultFilename);
+	if (resource.isHtml()) {
+		var htmlExtensions = resourceTypeExtensions[resourceTypes.html].possibleExtensions;
+		var resourceHasHtmlExtension = _.includes(htmlExtensions, extension);
+		// add index.html only if filepath has ext != html '/path/test.com' => '/path/test.com/index.html'
+		if (!resourceHasHtmlExtension) {
+			filePath = path.join(filePath, options.defaultFilename);
+		}
 	}
 
 	return sanitizeFilepath(filePath);

lib/filename-generators/by-type.js

@@ -1,7 +1,7 @@
 var _ = require('lodash');
 var path = require('path');
 var utils = require('../utils.js');
-var defaultExtensions = require('../config/resource-extensions-by-type');
+var typeExtensions = require('../config/resource-extensions-by-type');
 
 module.exports = function generateFilename (resource, options, occupiedFileNames) {
 	var occupiedNames = getSubDirectoryNames(options).concat(occupiedFileNames);
@@ -32,8 +32,8 @@ function getFilenameForResource (resource, options) {
 	var resourceType = resource.getType();
 	var extension = utils.getFilenameExtension(filename);
 
-	if (!extension && defaultExtensions[resourceType]) {
-		extension = defaultExtensions[resourceType];
+	if (!extension && typeExtensions[resourceType]) {
+		extension = typeExtensions[resourceType].defaultExtension;
 		filename += extension;
 	}
 

lib/utils.js

@@ -49,15 +49,22 @@ function getFilenameFromUrl (u) {
  * @returns {string} path
  */
 function getFilepathFromUrl (u) {
-	return url.parse(u).pathname.substring(1);
+	var normalizedUrl = normalizeUrl(u);
+	return url.parse(normalizedUrl).pathname.substring(1);
 }
 
 function getHashFromUrl (u) {
 	return url.parse(u).hash || '';
 }
 
-function getFilenameExtension (filename) {
-	return (typeof filename === 'string') ? path.extname(filename) : null;
+/**
+ * Returns extension for given filepath
+ * Example: some/path/file.js => .js
+ * @param {string} filepath
+ * @returns {string|null} - extension
+ */
+function getFilenameExtension (filepath) {
+	return (typeof filepath === 'string') ? path.extname(filepath) : null;
 }
 
 function shortenFilename (filename) {

test/unit/file-name-generators/by-site-structure-test.js

@@ -35,6 +35,14 @@ describe('byStructureFilenameGenerator', function() {
 		bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('index.html');
 	});
 
+	it('should add the defaultFilename to the path, for html resources with wrong extension', function(){
+		var isHtmlMock = sinon.stub().returns(true);
+
+		var r1 = new Resource('http://example.com/some/path/test.com');
+		r1.isHtml = isHtmlMock;
+		bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('some/path/test.com/index.html');
+	});
+
 	it('should normalize to safe relative paths, without ..', function(){
 		var r = new Resource('http://example.com/some/path/../../../../images/a.png');
 		bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('images/a.png');

test/unit/utils-test.js

@@ -77,6 +77,10 @@ describe('Common utils', function () {
 		it('should return path including filename if url has pathname', function() {
 			utils.getFilepathFromUrl('http://example.com/some/path/file.js').should.equal('some/path/file.js');
 		});
+		it('should not contain trailing slash', function() {
+			utils.getFilepathFromUrl('http://example.com/some/path/').should.equal('some/path');
+			utils.getFilepathFromUrl('http://example.com/some/path/file.css/').should.equal('some/path/file.css');
+		});
 	});
 
 	describe('#getHashFromUrl', function () {