Fix ignoring different string cases in extensions (#193)

Author: aivus

lib/scraper.js

@@ -22,6 +22,12 @@ function Scraper (options) {
 	self.options.request = _.extend({}, defaults.request, options.request);
 	self.options.urls = _.isArray(self.options.urls) ? self.options.urls : [self.options.urls];
 
+	if (self.options.subdirectories) {
+		self.options.subdirectories.forEach((element) => {
+			element.extensions = element.extensions.map((ext) => ext.toLowerCase());
+		});
+	}
+
 	if (self.options.recursive) {
 		self.options.sources = _.union(self.options.sources, recursiveSources);
 	}

lib/utils/index.js

@@ -78,7 +78,7 @@ function getHashFromUrl (u) {
  * @returns {string|null} - extension
  */
 function getFilenameExtension (filepath) {
-	return (typeof filepath === 'string') ? path.extname(filepath) : null;
+	return (typeof filepath === 'string') ? path.extname(filepath).toLowerCase() : null;
 }
 
 function shortenFilename (filename) {

test/unit/filename-generator/by-type-test.js

@@ -57,6 +57,18 @@ describe('FilenameGenerator: byType', function() {
 		filename.should.equalFileSystemPath('img/a.png');
 	});
 
+    it('should return filename with correct subdirectory when string cases are different', function() {
+        var options = {
+            subdirectories: [
+                { directory: 'img', extensions: ['.png'] }
+            ]
+        };
+
+        var r = new Resource('http://example.com/a.PNG');
+        var f = byTypeFilenameGenerator(r, options, []);
+        f.should.equalFileSystemPath('img/a.PNG');
+    });
+
 	it('should return different filename if desired filename is occupied', function() {
 		var r = new Resource('http://second-example.com/a.png');
 		var filename = byTypeFilenameGenerator(r, {}, [ 'a.png' ]);

test/unit/scraper-init-test.js

@@ -1,5 +1,5 @@
 var should = require('should');
-var proxyquire = require('proxyquire');
+var proxyquire = require('proxyquire').noCallThru();
 var sinon = require('sinon');
 var path = require('path');
 var Scraper = require('../../lib/scraper');
@@ -10,10 +10,10 @@ var urls = [ 'http://example.com' ];
 
 describe('Scraper initialization', function () {
 	describe('defaultFilename', function() {
-		var defaultsMock, Scraper;
+		var Scraper;
 
 		before(function() {
-			defaultsMock = { defaultFilename: 'dummyFilename.txt' };
+			var defaultsMock = { defaultFilename: 'dummyFilename.txt' };
 			Scraper = proxyquire('../../lib/scraper', {
 				'./config/defaults': defaultsMock
 			});
@@ -25,7 +25,7 @@ describe('Scraper initialization', function () {
 				directory: testDirname
 			});
 
-			s.options.defaultFilename.should.equalFileSystemPath(defaultsMock.defaultFilename);
+			s.options.defaultFilename.should.equalFileSystemPath('dummyFilename.txt');
 		});
 
 		it('should use defaultFilename sources if defaultFilename were passed', function () {
@@ -40,10 +40,10 @@ describe('Scraper initialization', function () {
 	});
 
 	describe('sources', function() {
-		var defaultsMock, Scraper;
+		var Scraper;
 
 		before(function() {
-			defaultsMock = { sources: ['1', '2', '3'] };
+			var defaultsMock = { sources: ['1', '2', '3'] };
 			Scraper = proxyquire('../../lib/scraper', {
 				'./config/defaults': defaultsMock
 			});
@@ -55,7 +55,7 @@ describe('Scraper initialization', function () {
 				directory: testDirname
 			});
 
-			s.options.sources.should.eql(defaultsMock.sources);
+			s.options.sources.should.eql(['1', '2', '3']);
 		});
 
 		it('should use passed sources if sources were passed', function () {
@@ -85,10 +85,10 @@ describe('Scraper initialization', function () {
 	});
 
 	describe('subdirectories', function () {
-		var defaultsMock, Scraper;
+		var Scraper;
 
 		before(function() {
-			defaultsMock = { directories: { directory: 'dir', extensions: ['.txt'] }, };
+			var defaultsMock = { subdirectories: [{ directory: 'dir', extensions: ['.txt'] }] };
 			Scraper = proxyquire('../../lib/scraper', {
 				'./config/defaults': defaultsMock
 			});
@@ -100,7 +100,20 @@ describe('Scraper initialization', function () {
 				directory: testDirname
 			});
 
-			s.options.subdirectories.should.eql(defaultsMock.subdirectories);
+			s.options.subdirectories.should.eql([{ directory: 'dir', extensions: ['.txt'] }]);
+		});
+
+		it('should convert extensions to lower case', function () {
+
+			var s = new Scraper({
+				urls: urls,
+				directory: testDirname,
+				subdirectories: [
+					{ directory: 'dir', extensions: ['.TXT'] }
+				]
+			});
+
+			s.options.subdirectories[0].extensions.should.eql(['.txt']);
 		});
 
 		it('should use passed subdirectories if subdirectories were passed', function () {
@@ -125,10 +138,10 @@ describe('Scraper initialization', function () {
 	});
 
 	describe('request', function () {
-		var defaultsMock, Scraper;
+		var Scraper;
 
 		before(function() {
-			defaultsMock = { request: { a: 1, b: 2 } };
+			var defaultsMock = { request: { a: 1, b: 2 } };
 			Scraper = proxyquire('../../lib/scraper', {
 				'./config/defaults': defaultsMock
 			});
@@ -140,7 +153,7 @@ describe('Scraper initialization', function () {
 				directory: testDirname
 			});
 
-			s.options.request.should.eql(defaultsMock.request);
+			s.options.request.should.eql({ a: 1, b: 2 });
 		});
 
 		it('should merge default and passed objects if request were passed', function () {