fix: various bug fixes for filesystem mode

Author: phawxby

README.md

@@ -58,6 +58,7 @@ scrape(options).then((result) => {});
 * [filenameGenerator](#filenamegenerator) - generate filename for downloaded resource
 * [requestConcurrency](#requestconcurrency) - set maximum concurrent requests
 * [tempMode](#tempMode) - How to store data temporarily during processing
+* [tempDir](#tempMode) - The directory to use to store temp files when `tempMode === fs`
 * [plugins](#plugins) - plugins, allow to customize filenames, request options, response handling, saving to storage, etc.
 
 Default options you can find in [lib/config/defaults.js](https://github.com/website-scraper/node-website-scraper/blob/master/lib/config/defaults.js) or get them using 
@@ -206,7 +207,7 @@ How to store temporary data when processing
 
 * `memory` - Data is store in memory in its raw format (default).
 * `memory-compressed` - Data is stored in memory but compressed using zlib. This is more memory efficient at the expense of CPU time spend compressing and decompressing.
-* `fs` / `filesystem` - Data is stored in temporary files on the filesystem. This is the most memory efficient but it is strongly recommended to only use this mode with a solid state drive.
+* `filesystem` - Data is stored in temporary files on the filesystem. This is the most memory efficient but it is strongly recommended to only use this mode with a solid state drive.
 
 #### plugins
 

lib/config/defaults.js

@@ -64,7 +64,8 @@ const config = {
 	maxRecursiveDepth: null,
 	maxDepth: null,
 	ignoreErrors: false,
-	tempMode: 'memory' // 'memory-compressed', 'fs'
+	tempMode: 'memory', // 'memory-compressed', 'fs'
+	tempDir: undefined
 };
 
 export default config;

lib/resource.js

@@ -4,16 +4,17 @@ import { promisify } from 'util';
 import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
-import os from 'os';
 
 const inflate = promisify(zlib.inflate);
 const defalate = promisify(zlib.deflate);
 
 class Resource {
-	constructor (url, filename, tmpMode, tmpDir) {
-		this.tmpMode = tmpMode || 'memory';
-		if (tmpMode === 'fs' || tmpMode === 'filesystem') {
-			this.tmpDir = tmpDir || fs.mkdtempSync(path.join(os.tmpdir(), 'website-scraper-'));
+	constructor (url, filename, tempMode, tempDir) {
+		this.tempMode = tempMode || 'memory';
+		this.tempDir = tempDir;
+
+		if (this.tempMode === 'filesystem' && !this.tempDir) {
+			throw new Error('tmpDir must be provided in tmpMode=filesystem');
 		}
 
 		this.setUrl(url);
@@ -30,7 +31,7 @@ class Resource {
 	}
 
 	createChild (url, filename) {
-		const child = new Resource(url, filename, this.tmpMode, this.tmpDir);
+		const child = new Resource(url, filename, this.tempMode, this.tempDir);
 		let currentDepth = this.getDepth();
 
 		child.parent = this;
@@ -53,10 +54,10 @@ class Resource {
 	}
 
 	setUrl (url) {
-		if (this.tmpDir) {
+		if (this.tempDir) {
 			// Generate a unique filename based on the md5 hash of the url
 			const tmpName = `${crypto.createHash('md5').update(url).digest('hex')}.txt`;
-			this.tmpPath = path.join(this.tmpDir, tmpName);
+			this.tmpPath = path.join(this.tempDir, tmpName);
 		}
 
 		this.url = url;
@@ -71,30 +72,28 @@ class Resource {
 	}
 
 	async getText () {
-		switch (this.tmpMode) {
+		switch (this.tempMode) {
 			case 'memory':
 				return await this._memoryRead();
 			case 'memory-compressed':
 				return await this._memoryReadCompressed();
-			case 'fs':
 			case 'filesystem':
 				return await this._fsRead();
 			default:
-				throw new Error('Unknown tmpMode');
+				throw new Error(`Unknown tempMode: ${this.tempMode}`);
 		}
 	}
 
 	async setText (text) {
-		switch (this.tmpMode) {
+		switch (this.tempMode) {
 			case 'memory':
 				return await this._memoryWrite(text);
 			case 'memory-compressed':
 				return await this._memoryWriteCompressed(text);
-			case 'fs':
 			case 'filesystem':
 				return await this._fsWrite(text);
 			default:
-				throw new Error('Unknown tmpMode');
+				throw new Error(`Unknown tempMode: ${this.tempMode}`);
 		}
 	}
 

lib/scraper.js

@@ -11,6 +11,9 @@ import {
 	GenerateFilenameByTypePlugin,
 	GetResourceReferencePlugin
 } from './plugins/index.js';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 
 import * as utils from './utils/index.js';
 const { extend, union, urlsEqual, getTypeByMime, getTypeByFilename, series } = utils;
@@ -47,7 +50,16 @@ class Scraper {
 			requestResource: this.requestResource.bind(this),
 			getReference: this.runActions.bind(this, 'getReference')
 		});
-		this.resources = this.options.urls.map(({url, filename}) => new Resource(url, filename, this.options.tmpMode));
+
+		logger.info('tmpMode', this.options.tempMode);
+		if (this.options.tempMode === 'filesystem') {
+			if (!this.options.tempDir) {
+				this.options.tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'website-scraper-'));
+			}
+			logger.info('tmpDir', this.options.tempDir);
+		}
+
+		this.resources = this.options.urls.map(({url, filename}) => new Resource(url, filename, this.options.tempMode, this.options.tempDir));
 
 		this.requestedResourcePromises = new NormalizedUrlMap(); // Map url -> request promise
 		this.loadedResources = new NormalizedUrlMap(); // Map url -> resource

test/unit/resource-test.js

@@ -2,6 +2,8 @@ import 'should';
 import Resource from '../../lib/resource.js';
 import fs from 'fs/promises';
 import '../utils/assertions.js';
+import path from 'path';
+import os from 'os';
 
 describe('Resource', () => {
 	describe('#createChild',  () => {
@@ -61,14 +63,11 @@ describe('Resource', () => {
 			(await resource.getText()).should.eql(testString2);
 		});
 
-		it('fs mode', async () => {
-			const resource = new Resource('http://example.com', 'index.html', 'fs');
-			resource.setEncoding('utf8');
-
-			resource.tmpDir.should.not.eql(undefined);
+		it('filesystem mode', async () => {
+			const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'website-scraper-'));
 
 			try {
-				const resource = new Resource('http://example.com', 'index.html', 'fs');
+				const resource = new Resource('http://example.com', 'index.html', 'filesystem', tmpDir);
 				resource.setEncoding('utf8');
 
 				await resource.setText(testString1);
@@ -77,7 +76,7 @@ describe('Resource', () => {
 				await resource.setText(testString2);
 				(await resource.getText()).should.eql(testString2);
 			} finally {
-				await fs.rm(resource.tmpDir, { recursive: true, force: true });
+				await fs.rm(tmpDir, { recursive: true, force: true });
 			}
 		});
 	});