Skip to content

Commit de9ee52

Browse files
jeromewujeromewu
committed
Use IDBFS and NODEFS to process big file
1 parent 91787bc commit de9ee52

File tree

20 files changed

+209
-156
lines changed

20 files changed

+209
-156
lines changed

‎examples/browser/image2video.html

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,13 @@ <h3>Click start to transcode images to mp4 (x264) and play!</h3>
3838
await worker.write(`tmp.${num}.png`, `../../tests/assets/triangle/tmp.${num}.png`);
3939
}
4040
message.innerHTML = 'Start transcoding';
41-
await worker.run('-framerate 30 -pattern_type glob -i *.png -i audio.ogg -c:a copy -shortest -c:v libx264 -pix_fmt yuv420p out.mp4');
41+
await worker.run('-framerate 30 -pattern_type glob -i /data/*.png -i /data/audio.ogg -c:a copy -shortest -c:v libx264 -pix_fmt yuv420p out.mp4', { outputPath: 'out.mp4' });
4242
const { data } = await worker.read('out.mp4');
43+
await worker.remove('audio.ogg');
44+
for (let i = 0; i < 60; i += 1) {
45+
const num = `00${i}`.slice(-3);
46+
await worker.remove(`tmp.${num}.png`);
47+
}
4348

4449
const video = document.getElementById('output-video');
4550
video.src = URL.createObjectURL(new Blob([data.buffer], { type: 'video/mp4' }));

‎examples/browser/run.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ <h3>Upload a video to transcode to mp4 (x264) and play!</h3>
3333
await worker.load();
3434
message.innerHTML = 'Start transcoding';
3535
await worker.write(name, files[0]);
36-
await worker.run(`-i ${name} output.mp4`);
36+
await worker.run(`-i /data/${name} output.mp4`, { inputPath: name, outputPath: 'output.mp4' });
3737
message.innerHTML = 'Complete transcoding';
3838
const { data } = await worker.read('output.mp4');
3939

‎examples/browser/transcode.html

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,20 @@ <h3>Upload a video to transcode to mp4 (x264) and play!</h3>
2323
const { createWorker } = FFmpeg;
2424
const worker = createWorker({
2525
corePath: '../../node_modules/@ffmpeg/core/ffmpeg-core.js',
26-
logger: ({ message }) => console.log(message),
26+
progress: p => console.log(p),
2727
});
2828

2929
const transcode = async ({ target: { files } }) => {
3030
const message = document.getElementById('message');
3131
const { name } = files[0];
3232
message.innerHTML = 'Loading ffmpeg-core.js';
3333
await worker.load();
34-
message.innerHTML = 'Start transcoding';
3534
await worker.write(name, files[0]);
35+
message.innerHTML = 'Start transcoding';
3636
await worker.transcode(name, 'output.mp4');
3737
message.innerHTML = 'Complete transcoding';
3838
const { data } = await worker.read('output.mp4');
39+
console.log(data);
3940

4041
const video = document.getElementById('output-video');
4142
video.src = URL.createObjectURL(new Blob([data.buffer], { type: 'video/mp4' }));

‎examples/node/image2video.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,14 @@ const worker = createWorker({
1515
await worker.write(`tmp.${num}.png`, `../../tests/assets/triangle/tmp.${num}.png`);
1616
}
1717
console.log('Start transcoding');
18-
await worker.run('-framerate 30 -pattern_type glob -i *.png -i audio.ogg -c:a copy -shortest -c:v libx264 -pix_fmt yuv420p out.mp4');
18+
await worker.run('-framerate 30 -pattern_type glob -i /data/*.png -i /data/audio.ogg -c:a copy -shortest -c:v libx264 -pix_fmt yuv420p out.mp4', { outputPath: 'out.mp4' });
1919
const { data } = await worker.read('out.mp4');
2020
console.log('Complete transcoding');
21+
await worker.remove('audio.ogg');
22+
for (let i = 0; i < 60; i += 1) {
23+
const num = `00${i}`.slice(-3);
24+
await worker.remove(`tmp.${num}.png`);
25+
}
2126
fs.writeFileSync('out.mp4', Buffer.from(data));
2227
process.exit(0);
2328
})();

‎examples/node/run.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const worker = createWorker({
99
await worker.load();
1010
console.log('Start transcoding');
1111
await worker.write('flame.avi', '../../tests/assets/flame.avi');
12-
await worker.run('-i flame.avi flame.mp4');
12+
await worker.run('-i /data/flame.avi flame.mp4', { inputPath: 'flame.avi', outputPath: 'flame.mp4' });
1313
const { data } = await worker.read('flame.mp4');
1414
console.log('Complete transcoding');
1515
fs.writeFileSync('flame.mp4', Buffer.from(data));

‎package-lock.json

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"lint": "eslint src",
1414
"wait": "rimraf dist && wait-on http://localhost:3000/dist/ffmpeg.dev.js",
1515
"test": "npm-run-all -p -r start test:all",
16-
"test:all": "npm-run-all wait test:browser:* test:node:all",
16+
"test:all": "npm-run-all wait test:node:all",
1717
"test:node": "nyc mocha --exit --bail --require ./scripts/test-helper.js",
1818
"test:node:all": "npm run test:node -- ./tests/*.test.js",
1919
"test:browser": "mocha-headless-chrome -a incognito -a no-sandbox -a disable-setuid-sandbox -a disable-logging -t 300000",
@@ -39,6 +39,7 @@
3939
"homepage": "https://github.com/ffmpegjs/ffmpeg.js#readme",
4040
"dependencies": {
4141
"@ffmpeg/core": "^0.4.0",
42+
"idb": "^4.0.5",
4243
"is-url": "^1.2.4",
4344
"node-fetch": "^2.6.0",
4445
"regenerator-runtime": "^0.13.3",

‎src/createWorker.js

Lines changed: 57 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ const {
88
spawnWorker,
99
terminateWorker,
1010
onMessage,
11-
loadMedia,
1211
send,
12+
fetchFile,
13+
fs,
1314
} = require('./worker/node');
1415

1516
let workerCounter = 0;
@@ -58,57 +59,72 @@ module.exports = (_options = {}) => {
5859
}))
5960
);
6061

61-
const write = async (path, data, jobId) => (
62+
const syncfs = (populate, jobId) => (
6263
startJob(createJob({
63-
id: jobId,
64-
action: 'write',
65-
payload: {
66-
path,
67-
data: await loadMedia(data),
68-
},
64+
id: jobId, action: 'syncfs', payload: { populate },
6965
}))
7066
);
7167

72-
const writeText = async (path, text, jobId) => (
73-
startJob(createJob({
74-
id: jobId,
75-
action: 'writeText',
76-
payload: {
77-
path,
78-
text,
79-
},
80-
}))
81-
);
68+
const write = async (path, data) => {
69+
await syncfs();
70+
await fs.writeFile(path, await fetchFile(data));
71+
await syncfs(true);
72+
return {
73+
path: `/data/${path}`,
74+
};
75+
};
8276

83-
const run = (args, jobId) => (
84-
startJob(createJob({
85-
id: jobId, action: 'run', payload: { args },
86-
}))
87-
);
77+
const writeText = async (path, text) => {
78+
await syncfs(true);
79+
await fs.writeFile(path, text);
80+
await syncfs(true);
81+
return {
82+
path: `/data/${path}`,
83+
};
84+
};
8885

89-
const transcode = (inputPath, outputPath, opts = '', jobId) => (
90-
run(`${opts} -i ${inputPath} ${outputPath}`, jobId)
91-
);
86+
const read = async (path, del = true) => {
87+
const data = await fs.readFile(path);
88+
if (del) {
89+
await fs.deleteFile(path);
90+
}
91+
return {
92+
data,
93+
};
94+
};
9295

93-
const trim = (inputPath, outputPath, from, to, opts = '', jobId) => (
94-
run(`${opts} -ss ${from} -i ${inputPath} -t ${to} -c copy ${outputPath}`, jobId)
95-
);
96+
const remove = async (path) => {
97+
await fs.deleteFile(path);
98+
return {
99+
path: `/data/${path}`,
100+
};
101+
};
96102

97-
const read = (path, jobId) => (
103+
const run = (args, opts = {}, jobId) => (
98104
startJob(createJob({
99-
id: jobId, action: 'read', payload: { path },
105+
id: jobId, action: 'run', payload: { args, options: opts },
100106
}))
101107
);
102108

103-
const remove = (path, jobId) => (
104-
startJob(createJob({
105-
id: jobId, action: 'remove', payload: { path },
106-
}))
109+
const transcode = (inputPath, outputPath, opts = '', del = true, jobId) => (
110+
run(
111+
`${opts} -i /data/${inputPath} ${outputPath}`,
112+
{ inputPath, outputPath, del },
113+
jobId,
114+
)
115+
);
116+
117+
const trim = (inputPath, outputPath, from, to, opts = '', del = true, jobId) => (
118+
run(
119+
`${opts} -ss ${from} -i /data/${inputPath} -t ${to} -c copy ${outputPath}`,
120+
{ inputPath, outputPath, del },
121+
jobId,
122+
)
107123
);
108124

109-
const mkdir = (path, jobId) => (
125+
const ls = (path, jobId) => (
110126
startJob(createJob({
111-
id: jobId, action: 'mkdir', payload: { path },
127+
id: jobId, action: 'ls', payload: { path },
112128
}))
113129
);
114130

@@ -151,14 +167,15 @@ module.exports = (_options = {}) => {
151167
setResolve,
152168
setReject,
153169
load,
170+
syncfs,
154171
write,
155172
writeText,
156-
transcode,
157-
trim,
158173
read,
159174
remove,
160-
mkdir,
161175
run,
176+
transcode,
177+
trim,
178+
ls,
162179
terminate,
163180
};
164181
};

‎src/utils/extractProgress.js

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,19 @@ const ts2sec = (ts) => {
66
};
77

88
module.exports = ({ message }, progress) => {
9-
if (message.startsWith(' Duration')) {
10-
const ts = message.split(', ')[0].split(': ')[1];
11-
const d = ts2sec(ts);
12-
if (duration === 0 || duration > d) {
13-
duration = d;
9+
if (typeof message === 'string') {
10+
if (message.startsWith(' Duration')) {
11+
const ts = message.split(', ')[0].split(': ')[1];
12+
const d = ts2sec(ts);
13+
if (duration === 0 || duration > d) {
14+
duration = d;
15+
}
16+
} else if (message.startsWith('frame')) {
17+
const ts = message.split('time=')[1].split(' ')[0];
18+
const t = ts2sec(ts);
19+
progress({ ratio: t / duration });
20+
} else if (message.startsWith('video:')) {
21+
progress({ ratio: 1 });
1422
}
15-
} else if (message.startsWith('frame')) {
16-
const ts = message.split('time=')[1].split(' ')[0];
17-
const t = ts2sec(ts);
18-
progress({ ratio: t / duration });
19-
} else if (message.startsWith('video:')) {
20-
progress({ ratio: 1 });
2123
}
2224
};

‎src/worker-script/browser/index.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
const worker = require('../');
22
const getCore = require('./getCore');
3+
const fs = require('../../worker/browser/fs');
34

45
global.addEventListener('message', ({ data }) => {
56
worker.dispatchHandlers(data, (obj) => postMessage(obj));
67
});
78

89
worker.setAdapter({
910
getCore,
11+
fs,
1012
});

0 commit comments

Comments
 (0)