Skip to content

Commit 529e27c

Browse files
s0ph1es0ph1e
authored
Use sanitizeFilaname in filename generators, fixes #413 (#425)
1 parent ae41620 commit 529e27c

File tree

6 files changed

+166
-127
lines changed

6 files changed

+166
-127
lines changed

‎lib/filename-generator/by-site-structure.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const _ = require('lodash');
22
const path = require('path');
33
const url = require('url');
4+
const sanitizeFilename = require('sanitize-filename');
45
const utils = require('../utils');
56
const resourceTypes = require('../config/resource-types');
67
const resourceTypeExtensions = require('../config/resource-ext-by-type');
@@ -12,7 +13,7 @@ module.exports = function generateFilename (resource, {defaultFilename}) {
1213
let filePath = utils.getFilepathFromUrl(resourceUrl);
1314
const extension = utils.getFilenameExtension(filePath);
1415

15-
filePath = path.join(host.replace(':', '_'), filePath);
16+
filePath = path.join(host, filePath);
1617

1718
// If have query string
1819
if (urlParsed.query) {
@@ -49,8 +50,7 @@ module.exports = function generateFilename (resource, {defaultFilename}) {
4950

5051
function sanitizeFilepath (filePath) {
5152
filePath = path.normalize(filePath);
52-
let pathParts = filePath.split(path.sep);
53-
pathParts = _.pull(pathParts, '..');
53+
const pathParts = filePath.split(path.sep).map(pathPart => sanitizeFilename(pathPart, {replacement: '_'}));
5454
pathParts[pathParts.length - 1] = utils.shortenFilename(_.last(pathParts));
5555
return pathParts.join(path.sep);
5656
}

‎lib/filename-generator/by-type.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
const _ = require('lodash');
22
const path = require('path');
3+
const sanitizeFilename = require('sanitize-filename');
34
const utils = require('../utils');
45
const typeExtensions = require('../config/resource-ext-by-type');
56

67
module.exports = function generateFilename (resource, {subdirectories, defaultFilename}, occupiedFileNames) {
78
const occupiedNames = getSubDirectoryNames({subdirectories}).concat(occupiedFileNames);
89

910
let filename = getFilenameForResource(resource, {subdirectories, defaultFilename});
10-
filename = utils.shortenFilename(filename);
11+
filename = utils.shortenFilename(sanitizeFilename(filename, {replacement: '_'}));
1112

1213
const extension = utils.getFilenameExtension(filename);
1314
const directory = getDirectoryByExtension(extension, {subdirectories, defaultFilename});

‎package-lock.json‎

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
"normalize-url": "^4.0.0",
4444
"p-queue": "^6.6.1",
4545
"request": "^2.85.0",
46+
"sanitize-filename": "^1.6.3",
4647
"srcset": "^2.0.0"
4748
},
4849
"devDependencies": {

‎test/unit/filename-generator/by-site-structure-test.js‎

Lines changed: 62 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,114 @@
1-
var _ = require('lodash');
2-
var should = require('should');
1+
const _ = require('lodash');
2+
const should = require('should');
33
require('../../utils/assertions');
4-
var sinon = require('sinon');
5-
var Resource = require('../../../lib/resource');
6-
var bySiteStructureFilenameGenerator = require('../../../lib/filename-generator/by-site-structure');
4+
const sinon = require('sinon');
5+
const Resource = require('../../../lib/resource');
6+
const bySiteStructureFilenameGenerator = require('../../../lib/filename-generator/by-site-structure');
77

8-
var options = { defaultFilename: 'index.html' };
8+
const options = { defaultFilename: 'index.html' };
99

10-
describe('FilenameGenerator: bySiteStructure', function() {
11-
it('should return the normalized relative path of the resource url', function(){
12-
var r1 = new Resource('http://example.com/some/path/a.png');
10+
describe('FilenameGenerator: bySiteStructure', () => {
11+
it('should return the normalized relative path of the resource url', () =>{
12+
const r1 = new Resource('http://example.com/some/path/a.png');
1313
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/a.png');
1414

15-
var r2 = new Resource('http://example.com/a.png');
15+
const r2 = new Resource('http://example.com/a.png');
1616
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/a.png');
17+
});
18+
19+
it('should remove . and .. from path', () => {
20+
const r1 = new Resource('http://example.com/some/path/../images/a.png');
21+
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/images/a.png');
22+
23+
const r2 = new Resource('http://example.com/some/path/../../../images/b.png');
24+
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/images/b.png');
1725

18-
var r3 = new Resource('http://example.com/some/path/../images/a.png');
19-
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/images/a.png');
26+
const r3 = new Resource('http://example.com/some/path/./images/c.png');
27+
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/path/images/c.png');
2028
});
2129

22-
it('should replace the colon, for url with port number', function(){
23-
var r1 = new Resource('http://example.com:8080/some/path/a.png');
30+
it('should replace the colon, for url with port number', () =>{
31+
const r1 = new Resource('http://example.com:8080/some/path/a.png');
2432
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com_8080/some/path/a.png');
2533
});
2634

27-
it('should add the defaultFilename to the path, for html resources without extension', function(){
28-
var isHtmlMock = sinon.stub().returns(true);
35+
it('should replace not allowed characters from filename', () => {
36+
const r1 = new Resource('http://example.com/some/path/<*a*>.png');
37+
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/__a__.png');
38+
});
39+
40+
it('should replace not allowed characters from path', () => {
41+
const r1 = new Resource('http://example.com/some:path/a.png');
42+
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some_path/a.png');
43+
});
44+
45+
it('should add the defaultFilename to the path, for html resources without extension', () =>{
46+
const isHtmlMock = sinon.stub().returns(true);
2947

30-
var r1 = new Resource('http://example.com/some/path/');
48+
const r1 = new Resource('http://example.com/some/path/');
3149
r1.isHtml = isHtmlMock;
3250
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/index.html');
3351

34-
var r2 = new Resource('http://example.com/some/path');
52+
const r2 = new Resource('http://example.com/some/path');
3553
r2.isHtml = isHtmlMock;
3654
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/some/path/index.html');
3755

38-
var r3 = new Resource('http://example.com');
56+
const r3 = new Resource('http://example.com');
3957
r3.isHtml = isHtmlMock;
4058
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/index.html');
4159
});
4260

43-
it('should add the defaultFilename to the path, for html resources with wrong extension', function(){
44-
var isHtmlMock = sinon.stub().returns(true);
61+
it('should add the defaultFilename to the path, for html resources with wrong extension', () =>{
62+
const isHtmlMock = sinon.stub().returns(true);
4563

46-
var r1 = new Resource('http://example.com/some/path/test.com');
64+
const r1 = new Resource('http://example.com/some/path/test.com');
4765
r1.isHtml = isHtmlMock;
4866
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/test.com/index.html');
4967
});
5068

51-
it('should normalize to safe relative paths, without ..', function(){
52-
var r = new Resource('http://example.com/some/path/../../../../images/a.png');
69+
it('should normalize to safe relative paths, without ..', () =>{
70+
const r = new Resource('http://example.com/some/path/../../../../images/a.png');
5371
bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/images/a.png');
5472
});
5573

56-
it('should not replace thrice dot in filenames', function() {
57-
// if it replaces them we receive 'some/path/../../../../etc/passwd'
58-
// path.resolve('some/path/../../../../etc/passwd'); = '/etc/passwd' => which is not safe
59-
var r = new Resource('http://example.com/some/path/.../.../.../.../etc/passwd');
60-
bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/some/path/.../.../.../.../etc/passwd');
61-
});
62-
63-
it('should shorten filename', function() {
64-
var resourceFilename = _.repeat('1', 1000) + '.png';
65-
var r = new Resource('http://example.com/' + resourceFilename);
66-
var filename = bySiteStructureFilenameGenerator(r, options);
74+
it('should shorten filename', () => {
75+
const resourceFilename = _.repeat('1', 1000) + '.png';
76+
const r = new Resource('http://example.com/' + resourceFilename);
77+
const filename = bySiteStructureFilenameGenerator(r, options);
6778
should(filename.length).be.lessThan(255);
6879
});
6980

70-
it('should shorten filename if resource is html without ext and default name is too long', function() {
71-
var defaultFilename = _.repeat('1', 1000) + '.html';
72-
var r = new Resource('http://example.com/path');
81+
it('should shorten filename if resource is html without ext and default name is too long', () => {
82+
const defaultFilename = _.repeat('1', 1000) + '.html';
83+
const r = new Resource('http://example.com/path');
7384
r.isHtml = sinon.stub().returns(true);
74-
var filepath = bySiteStructureFilenameGenerator(r, { defaultFilename: defaultFilename });
75-
var filename = _.last(filepath.split('/'));
85+
const filepath = bySiteStructureFilenameGenerator(r, { defaultFilename: defaultFilename });
86+
const filename = _.last(filepath.split('/'));
7687
should(filename.length).be.lessThan(255);
7788
});
7889

79-
it('should return decoded filepath', function() {
80-
var r = new Resource('https://developer.mozilla.org/ru/docs/JavaScript_%D1%88%D0%B5%D0%BB%D0%BB%D1%8B');
81-
var filename = bySiteStructureFilenameGenerator(r, options);
90+
it('should return decoded filepath', () => {
91+
const r = new Resource('https://developer.mozilla.org/ru/docs/JavaScript_%D1%88%D0%B5%D0%BB%D0%BB%D1%8B');
92+
const filename = bySiteStructureFilenameGenerator(r, options);
8293
filename.should.equalFileSystemPath('developer.mozilla.org/ru/docs/JavaScript_шеллы');
8394

84-
var r2 = new Resource('https://developer.mozilla.org/Hello%20G%C3%BCnter.png');
85-
var filename2 = bySiteStructureFilenameGenerator(r2, options);
95+
const r2 = new Resource('https://developer.mozilla.org/Hello%20G%C3%BCnter.png');
96+
const filename2 = bySiteStructureFilenameGenerator(r2, options);
8697
filename2.should.equalFileSystemPath('developer.mozilla.org/Hello Günter.png');
8798
});
8899

89-
it('should keep query strings', function () {
90-
var isHtmlMock = sinon.stub().returns(true);
100+
it('should keep query strings', () => {
101+
const isHtmlMock = sinon.stub().returns(true);
91102

92-
var r1 = new Resource('http://example.com/path?q=test');
103+
const r1 = new Resource('http://example.com/path?q=test');
93104
r1.isHtml = isHtmlMock;
94105
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/path/q=test.html');
95106

96-
var r2 = new Resource('http://example.com/path?q1=test1&q2=test2');
107+
const r2 = new Resource('http://example.com/path?q1=test1&q2=test2');
97108
r2.isHtml = isHtmlMock;
98109
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/path/q1=test1&q2=test2.html');
99110

100-
var r3 = new Resource('http://example.com/path/picture.png?q1=test1&q2=test2');
111+
const r3 = new Resource('http://example.com/path/picture.png?q1=test1&q2=test2');
101112
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/path/picture_q1=test1&q2=test2.png');
102-
})
113+
});
103114
});

0 commit comments

Comments
 (0)