added setting number of concurrent convert processes

Author: Maxime

src/converters/main.ts

@@ -1,4 +1,6 @@
-import { normalizeFiletype } from "../helpers/normalizeFiletype";
+import db from "../db/db";
+import { MAX_CONVERT_PROCESS } from "../helpers/env";
+import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
 import { convert as convertassimp, properties as propertiesassimp } from "./assimp";
 import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre";
 import { convert as convertDvisvgm, properties as propertiesDvisvgm } from "./dvisvgm";
@@ -105,6 +107,64 @@ const properties: Record<
   },
 };
 
+function chunks<T>(arr: T[], size: number): T[][] {
+  if(size <= 0){
+    return [arr]
+  }
+  return Array.from({ length: Math.ceil(arr.length / size) }, (_: T, i: number) =>
+    arr.slice(i * size, i * size + size)
+  );
+}
+
+export async function handleConvert(
+  fileNames: string[],
+  userUploadsDir: string,
+  userOutputDir: string,
+  convertTo: string,
+  converterName: string,
+  jobId: any
+) {
+  
+  const query = db.query(
+    "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
+  );
+
+
+  for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) {
+    for(const fileName of chunk) {
+      const toProcess: Promise<string>[] = [];
+      const filePath = `${userUploadsDir}${fileName}`;
+      const fileTypeOrig = fileName.split(".").pop() ?? "";
+      const fileType = normalizeFiletype(fileTypeOrig);
+      const newFileExt = normalizeOutputFiletype(convertTo);
+      const newFileName = fileName.replace(
+        new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
+        newFileExt,
+      );
+      const targetPath = `${userOutputDir}${newFileName}`;
+      toProcess.push(
+        new Promise((resolve, reject) => { 
+          mainConverter(
+              filePath,
+              fileType,
+              convertTo,
+              targetPath,
+              {},
+              converterName,
+            ).then(r =>  {
+              if (jobId.value) {
+                query.run(jobId.value, fileName, newFileName, r);
+              }
+              resolve(r);
+            }).catch(c => reject(c));
+        })
+      );
+      
+      await Promise.all(toProcess);
+    }
+  }
+}
+
 export async function mainConverter(
   inputFilePath: string,
   fileTypeOriginal: string,

src/helpers/env.ts

@@ -15,3 +15,5 @@ export const HIDE_HISTORY = process.env.HIDE_HISTORY?.toLowerCase() === "true" |
 export const WEBROOT = process.env.WEBROOT ?? "";
 
 export const LANGUAGE = process.env.LANGUAGE?.toLowerCase() || "en";
+
+export const MAX_CONVERT_PROCESS = process.env.MAX_CONVERT_PROCESS && Number(process.env.MAX_CONVERT_PROCESS) > 0 ? Number(process.env.MAX_CONVERT_PROCESS) : 0

src/pages/convert.tsx

@@ -2,11 +2,11 @@ import { mkdir } from "node:fs/promises";
 import { Elysia, t } from "elysia";
 import sanitize from "sanitize-filename";
 import { outputDir, uploadsDir } from "..";
-import { mainConverter } from "../converters/main";
+import { handleConvert } from "../converters/main";
 import db from "../db/db";
 import { Jobs } from "../db/types";
 import { WEBROOT } from "../helpers/env";
-import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
+import { normalizeFiletype } from "../helpers/normalizeFiletype";
 import { userService } from "./user";
 
 export const convert = new Elysia().use(userService).post(
@@ -61,36 +61,8 @@ export const convert = new Elysia().use(userService).post(
       jobId.value,
     );
 
-    const query = db.query(
-      "INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
-    );
-
     // Start the conversion process in the background
-    Promise.all(
-      fileNames.map(async (fileName) => {
-        const filePath = `${userUploadsDir}${fileName}`;
-        const fileTypeOrig = fileName.split(".").pop() ?? "";
-        const fileType = normalizeFiletype(fileTypeOrig);
-        const newFileExt = normalizeOutputFiletype(convertTo);
-        const newFileName = fileName.replace(
-          new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
-          newFileExt,
-        );
-        const targetPath = `${userOutputDir}${newFileName}`;
-
-        const result = await mainConverter(
-          filePath,
-          fileType,
-          convertTo,
-          targetPath,
-          {},
-          converterName,
-        );
-        if (jobId.value) {
-          query.run(jobId.value, fileName, newFileName, result);
-        }
-      }),
-    )
+    handleConvert(fileNames, userUploadsDir, userOutputDir, convertTo, converterName, jobId)
       .then(() => {
         // All conversions are done, update the job status to 'completed'
         if (jobId.value) {