Scrape resources with query strings when using bySiteStructure (#414), fixes #411

Author: Cerlancism

lib/filename-generator/by-site-structure.js

@@ -1,24 +1,46 @@
 const _ = require('lodash');
 const path = require('path');
+const url = require('url');
 const utils = require('../utils');
 const resourceTypes = require('../config/resource-types');
 const resourceTypeExtensions = require('../config/resource-ext-by-type');
 
 module.exports = function generateFilename (resource, {defaultFilename}) {
 	const resourceUrl = resource.getUrl();
 	const host = utils.getHostFromUrl(resourceUrl);
+	const urlParsed = url.parse(resourceUrl);
 	let filePath = utils.getFilepathFromUrl(resourceUrl);
 	const extension = utils.getFilenameExtension(filePath);
 
 	filePath = path.join(host.replace(':', '_'), filePath);
 
+	// If have query string
+	if (urlParsed.query) {
+		const parsed = path.parse(filePath);
+		const basename = path.join(parsed.dir, parsed.name);
+		// Use the query string as file name in the site structure directory
+		if (!extension) {
+			// Without extension: http://example.com/path?q=test => path/q=test
+			filePath = `${basename}${path.sep}${urlParsed.query}`;
+		} else {
+			// With extension: http://example.com/path/picture.png?q=test => path/picture_q=test.png
+			filePath = `${basename}_${urlParsed.query}${extension}`;
+		}
+	}
+
 	// If we have HTML from 'http://example.com/path' => set 'path/index.html' as filepath
 	if (resource.isHtml()) {
 		const htmlExtensions = resourceTypeExtensions[resourceTypes.html];
 		const resourceHasHtmlExtension = _.includes(htmlExtensions, extension);
 		// add index.html only if filepath has ext != html '/path/test.com' => '/path/test.com/index.html'
 		if (!resourceHasHtmlExtension) {
-			filePath = path.join(filePath, defaultFilename);
+			if (!urlParsed.query) {
+				// Without query string: http://example.com/path => path/index.html
+				filePath = path.join(filePath, defaultFilename);
+			} else {
+				// With query string: http://example.com/path?q=test => path/q=test.html
+				filePath = `${filePath}.html`;
+			}
 		}
 	}
 

test/unit/filename-generator/by-site-structure-test.js

@@ -85,4 +85,19 @@ describe('FilenameGenerator: bySiteStructure', function() {
 		var filename2 = bySiteStructureFilenameGenerator(r2, options);
 		filename2.should.equalFileSystemPath('developer.mozilla.org/Hello Günter.png');
 	});
+
+	it('should keep query strings', function () {
+		var isHtmlMock = sinon.stub().returns(true);
+
+		var r1 = new Resource('http://example.com/path?q=test');
+		r1.isHtml = isHtmlMock;
+		bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/path/q=test.html');
+
+		var r2 = new Resource('http://example.com/path?q1=test1&q2=test2');
+		r2.isHtml = isHtmlMock;
+		bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/path/q1=test1&q2=test2.html');
+
+		var r3 = new Resource('http://example.com/path/picture.png?q1=test1&q2=test2');
+		bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/path/picture_q1=test1&q2=test2.png');
+	})
 });