Skip to content

Commit a9bc9d7

Browse files
C4illinC4illin
authored
Merge pull request #342 from Netzz0/FEAT/better-handling-of-multiples-files
Better handling of multiples files (Added Archive downloads and env var to set maximum concurrent processes)
2 parents 18fed70 + 4ad7892 commit a9bc9d7

File tree

8 files changed

+96
-62
lines changed

8 files changed

+96
-62
lines changed

‎bun.lock‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"@kitajs/html": "^4.2.9",
1111
"elysia": "^1.3.4",
1212
"sanitize-filename": "^1.6.3",
13+
"tar": "^7.4.3",
1314
},
1415
"devDependencies": {
1516
"@eslint/js": "^9.28.0",

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
"@elysiajs/static": "^1.3.0",
2121
"@kitajs/html": "^4.2.9",
2222
"elysia": "^1.3.4",
23-
"sanitize-filename": "^1.6.3"
23+
"sanitize-filename": "^1.6.3",
24+
"tar": "^7.4.3"
2425
},
2526
"module": "src/index.tsx",
2627
"type": "module",

‎public/results.js‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,4 @@
11
const webroot = document.querySelector("meta[name='webroot']").content;
2-
3-
window.downloadAll = function () {
4-
// Get all download links
5-
const downloadLinks = document.querySelectorAll("a[download]");
6-
7-
// Trigger download for each link
8-
downloadLinks.forEach((link, index) => {
9-
// We add a delay for each download to prevent them from starting at the same time
10-
setTimeout(() => {
11-
const event = new MouseEvent("click");
12-
link.dispatchEvent(event);
13-
}, index * 100);
14-
});
15-
};
162
const jobId = window.location.pathname.split("/").pop();
173
const main = document.querySelector("main");
184
let progressElem = document.querySelector("progress");

‎src/converters/main.ts‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
import { normalizeFiletype } from "../helpers/normalizeFiletype";
1+
import db from "../db/db";
2+
import { MAX_CONVERT_PROCESS } from "../helpers/env";
3+
import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
24
import { convert as convertassimp, properties as propertiesassimp } from "./assimp";
35
import { convert as convertCalibre, properties as propertiesCalibre } from "./calibre";
46
import { convert as convertDvisvgm, properties as propertiesDvisvgm } from "./dvisvgm";
@@ -111,6 +113,63 @@ const properties: Record<
111113
},
112114
};
113115

116+
function chunks<T>(arr: T[], size: number): T[][] {
117+
if(size <= 0){
118+
return [arr]
119+
}
120+
return Array.from({ length: Math.ceil(arr.length / size) }, (_: T, i: number) =>
121+
arr.slice(i * size, i * size + size)
122+
);
123+
}
124+
125+
export async function handleConvert(
126+
fileNames: string[],
127+
userUploadsDir: string,
128+
userOutputDir: string,
129+
convertTo: string,
130+
converterName: string,
131+
jobId: any
132+
) {
133+
134+
const query = db.query(
135+
"INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
136+
);
137+
138+
139+
for (const chunk of chunks(fileNames, MAX_CONVERT_PROCESS)) {
140+
const toProcess: Promise<string>[] = [];
141+
for(const fileName of chunk) {
142+
const filePath = `${userUploadsDir}${fileName}`;
143+
const fileTypeOrig = fileName.split(".").pop() ?? "";
144+
const fileType = normalizeFiletype(fileTypeOrig);
145+
const newFileExt = normalizeOutputFiletype(convertTo);
146+
const newFileName = fileName.replace(
147+
new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
148+
newFileExt,
149+
);
150+
const targetPath = `${userOutputDir}${newFileName}`;
151+
toProcess.push(
152+
new Promise((resolve, reject) => {
153+
mainConverter(
154+
filePath,
155+
fileType,
156+
convertTo,
157+
targetPath,
158+
{},
159+
converterName,
160+
).then(r => {
161+
if (jobId.value) {
162+
query.run(jobId.value, fileName, newFileName, r);
163+
}
164+
resolve(r);
165+
}).catch(c => reject(c));
166+
})
167+
);
168+
}
169+
await Promise.all(toProcess);
170+
}
171+
}
172+
114173
export async function mainConverter(
115174
inputFilePath: string,
116175
fileTypeOriginal: string,

‎src/helpers/env.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,5 @@ export const HIDE_HISTORY = process.env.HIDE_HISTORY?.toLowerCase() === "true" |
1515
export const WEBROOT = process.env.WEBROOT ?? "";
1616

1717
export const LANGUAGE = process.env.LANGUAGE?.toLowerCase() || "en";
18+
19+
export const MAX_CONVERT_PROCESS = process.env.MAX_CONVERT_PROCESS && Number(process.env.MAX_CONVERT_PROCESS) > 0 ? Number(process.env.MAX_CONVERT_PROCESS) : 0

‎src/pages/convert.tsx‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ import { mkdir } from "node:fs/promises";
22
import { Elysia, t } from "elysia";
33
import sanitize from "sanitize-filename";
44
import { outputDir, uploadsDir } from "..";
5-
import { mainConverter } from "../converters/main";
5+
import { handleConvert } from "../converters/main";
66
import db from "../db/db";
77
import { Jobs } from "../db/types";
88
import { WEBROOT } from "../helpers/env";
9-
import { normalizeFiletype, normalizeOutputFiletype } from "../helpers/normalizeFiletype";
9+
import { normalizeFiletype } from "../helpers/normalizeFiletype";
1010
import { userService } from "./user";
1111

1212
export const convert = new Elysia().use(userService).post(
@@ -61,36 +61,8 @@ export const convert = new Elysia().use(userService).post(
6161
jobId.value,
6262
);
6363

64-
const query = db.query(
65-
"INSERT INTO file_names (job_id, file_name, output_file_name, status) VALUES (?1, ?2, ?3, ?4)",
66-
);
67-
6864
// Start the conversion process in the background
69-
Promise.all(
70-
fileNames.map(async (fileName) => {
71-
const filePath = `${userUploadsDir}${fileName}`;
72-
const fileTypeOrig = fileName.split(".").pop() ?? "";
73-
const fileType = normalizeFiletype(fileTypeOrig);
74-
const newFileExt = normalizeOutputFiletype(convertTo);
75-
const newFileName = fileName.replace(
76-
new RegExp(`${fileTypeOrig}(?!.*${fileTypeOrig})`),
77-
newFileExt,
78-
);
79-
const targetPath = `${userOutputDir}${newFileName}`;
80-
81-
const result = await mainConverter(
82-
filePath,
83-
fileType,
84-
convertTo,
85-
targetPath,
86-
{},
87-
converterName,
88-
);
89-
if (jobId.value) {
90-
query.run(jobId.value, fileName, newFileName, result);
91-
}
92-
}),
93-
)
65+
handleConvert(fileNames, userUploadsDir, userOutputDir, convertTo, converterName, jobId)
9466
.then(() => {
9567
// All conversions are done, update the job status to 'completed'
9668
if (jobId.value) {

‎src/pages/download.tsx‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { outputDir } from "..";
44
import db from "../db/db";
55
import { WEBROOT } from "../helpers/env";
66
import { userService } from "./user";
7+
import path from "node:path";
8+
import * as tar from "tar";
79

810
export const download = new Elysia()
911
.use(userService)
@@ -35,8 +37,7 @@ export const download = new Elysia()
3537
return Bun.file(filePath);
3638
},
3739
)
38-
.get("/zip/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => {
39-
// TODO: Implement zip download
40+
.get("/archive/:userId/:jobId", async ({ params, jwt, redirect, cookie: { auth } }) => {
4041
if (!auth?.value) {
4142
return redirect(`${WEBROOT}/login`, 302);
4243
}
@@ -54,9 +55,11 @@ export const download = new Elysia()
5455
return redirect(`${WEBROOT}/results`, 302);
5556
}
5657

57-
// const userId = decodeURIComponent(params.userId);
58-
// const jobId = decodeURIComponent(params.jobId);
59-
// const outputPath = `${outputDir}${userId}/`{jobId}/);
58+
const userId = decodeURIComponent(params.userId);
59+
const jobId = decodeURIComponent(params.jobId);
60+
const outputPath = `${outputDir}${userId}/${jobId}`;
61+
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`)
6062

61-
// return Bun.zip(outputPath);
63+
await tar.create({file: outputTar, cwd: outputPath, filter: (path) => { return !path.match(".*\\.tar"); }}, ["."]);
64+
return Bun.file(outputTar);
6265
});

‎src/pages/results.tsx‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,17 @@ import db from "../db/db";
66
import { Filename, Jobs } from "../db/types";
77
import { ALLOW_UNAUTHENTICATED, WEBROOT } from "../helpers/env";
88
import { userService } from "./user";
9+
import { JWTPayloadSpec } from "@elysiajs/jwt";
910

1011
function ResultsArticle({
12+
user,
1113
job,
1214
files,
1315
outputPath,
1416
}: {
17+
user: {
18+
id: string;
19+
} & JWTPayloadSpec;
1520
job: Jobs;
1621
files: Filename[];
1722
outputPath: string;
@@ -21,14 +26,19 @@ function ResultsArticle({
2126
<div class="mb-4 flex items-center justify-between">
2227
<h1 class="text-xl">Results</h1>
2328
<div>
24-
<button
25-
type="button"
26-
class="float-right w-40 btn-primary"
27-
onclick="downloadAll()"
28-
{...(files.length !== job.num_files ? { disabled: true, "aria-busy": "true" } : "")}
29+
<a
30+
style={files.length !== job.num_files ? "pointer-events: none;" : ""}
31+
href={`${WEBROOT}/archive/${user.id}/${job.id}`}
32+
download={`converted_files_${job.id}.tar`}
2933
>
30-
{files.length === job.num_files ? "Download All" : "Converting..."}
31-
</button>
34+
<button
35+
type="button"
36+
class="float-right w-40 btn-primary"
37+
{...(files.length !== job.num_files ? { disabled: true, "aria-busy": "true" } : "")}
38+
>
39+
{files.length === job.num_files ? "Download All" : "Converting..."}
40+
</button>
41+
</a>
3242
</div>
3343
</div>
3444
<progress
@@ -170,7 +180,7 @@ export const results = new Elysia()
170180
sm:px-4
171181
`}
172182
>
173-
<ResultsArticle job={job} files={files} outputPath={outputPath} />
183+
<ResultsArticle user={user} job={job} files={files} outputPath={outputPath} />
174184
</main>
175185
<script src={`${WEBROOT}/results.js`} defer />
176186
</>
@@ -211,5 +221,5 @@ export const results = new Elysia()
211221
.as(Filename)
212222
.all(params.jobId);
213223

214-
return <ResultsArticle job={job} files={files} outputPath={outputPath} />;
224+
return <ResultsArticle user={user} job={job} files={files} outputPath={outputPath} />;
215225
});

0 commit comments

Comments
 (0)