Add missing ext for html and css in byType generator, close #58 (#59)

* Add missing ext for html and css in byType generator, close #58

* Add EOL

Author: s0ph1e

lib/config/resource-extensions-by-type.js

@@ -0,0 +1,8 @@
+var types = require('./resource-types');
+
+var defaultExtensions = {};
+
+defaultExtensions[types.html] = '.html';
+defaultExtensions[types.css] = '.css';
+
+module.exports = defaultExtensions;

lib/filename-generators/by-type.js

@@ -1,16 +1,15 @@
 var _ = require('lodash');
 var path = require('path');
 var utils = require('../utils.js');
+var defaultExtensions = require('../config/resource-extensions-by-type');
 
 module.exports = function generateFilename (resource, options, loadedResources) {
 	var occupiedFilenames = getOccupiedFilenames(loadedResources, options);
 
-	var preferredFilename = resource.getFilename();
-	var urlFilename = utils.getFilenameFromUrl(resource.getUrl());
-	var filename = preferredFilename || urlFilename || options.defaultFilename;
-
+	var filename = getFilenameForResource(resource, options);
 	var extension = path.extname(filename);
 	var directory = getDirectoryByExtension(extension, options);
+
 	var currentFilename = path.join(directory, filename);
 	var basename = path.basename(filename, extension);
 	var index = 1;
@@ -22,6 +21,22 @@ module.exports = function generateFilename (resource, options, loadedResources)
 	return currentFilename;
 };
 
+function getFilenameForResource (resource, options) {
+	var preferredFilename = resource.getFilename();
+	var urlFilename = utils.getFilenameFromUrl(resource.getUrl());
+	var filename = preferredFilename || urlFilename || options.defaultFilename;
+
+	var resourceType = resource.getType();
+	var extension = path.extname(filename);
+
+	if (!extension && defaultExtensions[resourceType]) {
+		extension = defaultExtensions[resourceType];
+		filename += extension;
+	}
+
+	return filename;
+}
+
 function getOccupiedFilenames (loadedResources, options) {
 	var subdirectories = _.map(options.subdirectories, function getDirectory (directory) { return directory.directory; });
 	var loadedFiles = _.map(loadedResources, function getFileName (resource) { return resource.getFilename(); });

test/functional/resources-without-ext-test.js

@@ -19,7 +19,7 @@ describe('Functional resources without extensions', function() {
 		fs.removeSync(testDirname);
 	});
 
-	it('should load resources without extensions with correct type', function(done) {
+	it('should load resources without extensions with correct type and wrap with extensions', function(done) {
 		var options = {
 			urls: [ 'http://example.com/' ],
 			directory: testDirname,
@@ -49,7 +49,7 @@ describe('Functional resources without extensions', function() {
 
 		scraper.scrape(options).then(function() {
 			// should load css file and fonts from css file
-			fs.existsSync(testDirname + '/css').should.be.eql(true); // http://fonts.googleapis.com/css?family=Lato
+			fs.existsSync(testDirname + '/css.css').should.be.eql(true); // http://fonts.googleapis.com/css?family=Lato
 			fs.existsSync(testDirname + '/UyBMtLsHKBKXelqf4x7VRQ.woff2').should.be.eql(true);
 			fs.existsSync(testDirname + '/1YwB1sO8YE1Lyjf12WNiUA.woff2').should.be.eql(true);
 
@@ -58,7 +58,7 @@ describe('Functional resources without extensions', function() {
 			fs.existsSync(testDirname + '/google.png').should.be.eql(true);
 
 			// should load html file and its sources from iframe
-			fs.existsSync(testDirname + '/iframe').should.be.eql(true);
+			fs.existsSync(testDirname + '/iframe.html').should.be.eql(true);
 			fs.existsSync(testDirname + '/cat.png').should.be.eql(true);
 
 			done();

test/unit/file-name-generators/by-type-test.js

@@ -1,4 +1,5 @@
 require('should');
+var sinon = require('sinon');
 var Scraper = require('../../../lib/scraper');
 var Resource = require('../../../lib/resource');
 var byTypeFilenameGenerator = require('../../../lib/filename-generators/by-type');
@@ -36,6 +37,54 @@ describe('byTypeFilenameGenerator', function() {
         }).catch(done);
     });
 
+    it('should add missed extensions for html resources', function (done) {
+        var s = new Scraper({
+            urls: 'http://example.com',
+            directory: testDirname,
+            subdirectories: null
+        });
+
+        s.prepare().then(function() {
+            var r = new Resource('http://example.com/about', '');
+            r.getType = sinon.stub().returns('html');
+            var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources);
+            filename.should.be.eql('about.html');
+            done();
+        }).catch(done);
+    });
+
+    it('should add missed extensions for css resources', function (done) {
+        var s = new Scraper({
+            urls: 'http://example.com',
+            directory: testDirname,
+            subdirectories: null
+        });
+
+        s.prepare().then(function() {
+            var r = new Resource('http://example.com/css', '');
+            r.getType = sinon.stub().returns('css');
+            var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources);
+            filename.should.be.eql('css.css');
+            done();
+        }).catch(done);
+    });
+
+    it('should not add missed extensions for other resources', function (done) {
+        var s = new Scraper({
+            urls: 'http://example.com',
+            directory: testDirname,
+            subdirectories: null
+        });
+
+        s.prepare().then(function() {
+            var r = new Resource('http://1.gravatar.com/avatar/4d63e4a045c7ff22accc33dc08442f86?s=140&d=%2Fwp-content%2Fuploads%2F2015%2F05%2FGood-JOb-150x150.jpg&r=g', '');
+            r.getType = sinon.stub().returns('home');
+            var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources);
+            filename.should.be.eql('4d63e4a045c7ff22accc33dc08442f86');
+            done();
+        }).catch(done);
+    });
+
     it('should return filename with correct subdirectory', function(done) {
         var s = new Scraper({
             urls: 'http://example.com',