ImportEtherpad: Limit in-flight DB queries

Author: rhansen

src/node/utils/ImportEtherpad.js

@@ -18,6 +18,7 @@
 
 const AttributePool = require('../../static/js/AttributePool');
 const {Pad} = require('../db/Pad');
+const async = require('async');
 const authorManager = require('../db/AuthorManager');
 const db = require('../db/DB');
 const hooks = require('../../static/js/pluginfw/hooks');
@@ -47,12 +48,16 @@ exports.setPadRaw = async (padId, r) => {
     if (originalPadId !== padId) throw new Error('unexpected pad ID in record');
   };
 
+  // Limit the number of in-flight database queries so that the queries do not time out when
+  // importing really large files.
+  const q = async.queue(async (task) => await task(), 100);
+
   // First validate and transform values. Do not commit any records to the database yet in case
   // there is a problem with the data.
 
   const dbRecords = new Map();
   const existingAuthors = new Set();
-  await Promise.all(Object.entries(records).map(async ([key, value]) => {
+  await Promise.all(Object.entries(records).map(([key, value]) => q.pushAsync(async () => {
     if (!value) {
       return;
     }
@@ -91,7 +96,7 @@ exports.setPadRaw = async (padId, r) => {
       return;
     }
     dbRecords.set(key, value);
-  }));
+  })));
 
   const pad = new Pad(padId, {
     // Only fetchers are needed to check the pad's integrity.
@@ -109,7 +114,7 @@ exports.setPadRaw = async (padId, r) => {
   await pad.check();
 
   await Promise.all([
-    ...[...dbRecords].map(async ([k, v]) => await db.set(k, v)),
-    ...[...existingAuthors].map(async (authorId) => await authorManager.addPad(authorId, padId)),
+    ...[...dbRecords].map(([k, v]) => q.pushAsync(() => db.set(k, v))),
+    ...[...existingAuthors].map((a) => q.pushAsync(() => authorManager.addPad(a, padId))),
   ]);
 };