Skip to content

Commit ff456f0

Browse files
authored
Merge pull request #24 from shiftonetothree/fix/fix-audio-memory-leak
Fix/fix audio memory leak
2 parents 672bd12 + 64f580e commit ff456f0

File tree

7 files changed

+136
-12
lines changed

7 files changed

+136
-12
lines changed

src/cubism-common/MotionManager.ts

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -257,8 +257,6 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends util
257257
let soundURL: string | undefined;
258258
const isBase64Content = sound && sound.startsWith("data:");
259259

260-
console.log(onFinish)
261-
262260
if (sound && !isBase64Content) {
263261
const A = document.createElement("a");
264262
A.href = sound;
@@ -275,7 +273,7 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends util
275273
audio = SoundManager.add(
276274
file,
277275
(that = this) => {
278-
console.log('Audio finished playing'); // Add this line
276+
console.log("Audio finished playing"); // Add this line
279277
onFinish?.();
280278
resetExpression &&
281279
expression &&
@@ -284,7 +282,7 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends util
284282
that.currentAudio = undefined;
285283
}, // reset expression when audio is done
286284
(e, that = this) => {
287-
console.log('Error during audio playback:', e); // Add this line
285+
console.log("Error during audio playback:", e); // Add this line
288286
onError?.(e);
289287
resetExpression &&
290288
expression &&
@@ -425,17 +423,16 @@ export abstract class MotionManager<Motion = any, MotionSpec = any> extends util
425423
audio = SoundManager.add(
426424
file,
427425
(that = this) => {
428-
console.log('Audio finished playing'); // Add this line
426+
console.log("Audio finished playing"); // Add this line
429427
onFinish?.();
430-
console.log(onFinish)
431428
resetExpression &&
432429
expression &&
433430
that.expressionManager &&
434431
that.expressionManager.resetExpression();
435432
that.currentAudio = undefined;
436433
}, // reset expression when audio is done
437434
(e, that = this) => {
438-
console.log('Error during audio playback:', e); // Add this line
435+
console.log("Error during audio playback:", e); // Add this line
439436
onError?.(e);
440437
resetExpression &&
441438
expression &&

src/cubism-common/SoundManager.ts

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ const TAG = "SoundManager";
44
export const VOLUME = 0.5;
55

66
const audioListenersWeakMap = new WeakMap();
7+
const audioCanplaythroughWeakMap = new WeakMap();
8+
const audioContextWeakMap = new WeakMap<HTMLAudioElement, AudioContext>();
9+
const audioAnalyserWeakMap = new WeakMap<HTMLAudioElement, AnalyserNode>();
10+
const audioSourceWeakMap = new WeakMap<HTMLAudioElement, MediaElementAudioSourceNode>();
711
/**
812
* Manages all the sounds.
913
*/
@@ -85,6 +89,7 @@ export class SoundManager {
8589
if (audio.readyState === audio.HAVE_ENOUGH_DATA) {
8690
resolve();
8791
} else {
92+
audioCanplaythroughWeakMap.set(audio, resolve);
8893
audio.addEventListener("canplaythrough", resolve as () => void);
8994
}
9095
});
@@ -93,6 +98,7 @@ export class SoundManager {
9398
static addContext(audio: HTMLAudioElement): AudioContext {
9499
/* Create an AudioContext */
95100
const context = new AudioContext();
101+
audioContextWeakMap.set(audio, context);
96102

97103
this.contexts.push(context);
98104
return context;
@@ -111,6 +117,8 @@ export class SoundManager {
111117
source.connect(analyser);
112118
analyser.connect(context.destination);
113119

120+
audioSourceWeakMap.set(audio, source);
121+
audioAnalyserWeakMap.set(audio, analyser);
114122
this.analysers.push(analyser);
115123
return analyser;
116124
}
@@ -143,9 +151,21 @@ export class SoundManager {
143151
audio.pause();
144152
audio.removeEventListener("ended", audioListenersWeakMap.get(audio)?.ended);
145153
audio.removeEventListener("error", audioListenersWeakMap.get(audio)?.error);
154+
audio.removeEventListener("canplaythrough", audioCanplaythroughWeakMap.get(audio));
146155
audioListenersWeakMap.delete(audio);
156+
audioCanplaythroughWeakMap.delete(audio);
157+
const context = audioContextWeakMap.get(audio);
158+
audioContextWeakMap.delete(audio);
159+
context?.close();
160+
const analyser = audioAnalyserWeakMap.get(audio);
161+
audioAnalyserWeakMap.delete(audio);
162+
analyser?.disconnect();
163+
const source = audioSourceWeakMap.get(audio);
164+
audioSourceWeakMap.delete(audio);
165+
source?.disconnect();
147166
audio.removeAttribute("src");
148-
167+
remove(this.analysers, analyser);
168+
remove(this.contexts, context);
149169
remove(this.audios, audio);
150170
}
151171

test/env.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,10 @@ type TestModel = (typeof ALL_TEST_MODELS)[number];
9999
export function testEachModel(
100100
name: string,
101101
fn: (ctx: TestContext & CustomContext & { model: TestModel }) => Awaitable<void>,
102+
options?: number,
102103
) {
103104
for (const model of ALL_TEST_MODELS) {
104-
test.extend({ model })(`${name} (${model.name})`, fn as any);
105+
test.extend({ model })(`${name} (${model.name})`, fn as any, options);
105106
}
106107
}
107108

test/features/stability.test.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { testEachModel } from "../env";
2+
import { createModel } from "../utils";
3+
import { makeTestSound } from "../make-test-sound";
4+
5+
testEachModel(
6+
"speak 1000 times",
7+
async ({ app, model: { modelJsonWithUrl, hitTests } }) => {
8+
const model = await createModel(modelJsonWithUrl);
9+
model.update(100);
10+
app.stage.addChild(model);
11+
app.renderer.resize(model.width, model.height);
12+
app.render();
13+
for (let i = 0; i < 1000; i++) {
14+
await new Promise<void>((resolve, reject) => {
15+
const duration = Math.random() * 0.01;
16+
model.speak(makeTestSound(undefined, 0.1, duration), {
17+
onFinish: resolve,
18+
onError: reject,
19+
});
20+
});
21+
}
22+
},
23+
2 * 60 * 1000,
24+
);

test/make-test-sound.d.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
declare function makeTestSound(freq?:number,volume?:number,duration?:number,fadeout?:number): string;
2+
export { makeTestSound };

test/make-test-sound.js

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// this code from https://gist.github.com/vanyle/c31dcd43d450543e423f8fa745ae67e2
2+
3+
// This function only generates pure frequencies but you can customize the waveform
4+
// by changing the function at line ~50 (the one with the cos)
5+
6+
export function makeTestSound(freq, volume, duration, fadeout) {
7+
// Generate a wav binary file, convert it to a blob and play it.
8+
// https://fr.wikipedia.org/wiki/Waveform_Audio_File_Format#Structure_des_fichiers_WAV
9+
10+
const sampling = 44100; // in hertz
11+
duration = duration || 1; // in seconds
12+
const sampleCount = Math.floor(sampling * duration);
13+
14+
volume = volume || 0.5;
15+
if (volume > 1) volume = 1;
16+
if (volume < 0) volume = 0; // clamp
17+
volume = Math.floor(volume * 256 * 256); // convert to 16 bit integer
18+
19+
freq = freq || 440; // nice default freq
20+
21+
// Helper function to build binary files
22+
function toBytes(value, bytes) {
23+
let result = "";
24+
for (; bytes > 0; bytes--) {
25+
result += String.fromCharCode(value & 255);
26+
value >>= 8;
27+
}
28+
return result;
29+
}
30+
31+
let samples = "";
32+
33+
// Example of a fade: linear fade.
34+
// fadeout = fadeout || ((t,max) => (max-t)/max);
35+
36+
// By default, be use a fancy "fast attack, slow descent" fade
37+
fadeout =
38+
fadeout ||
39+
((t, max) => {
40+
const attackRatio = 0.01;
41+
if (t / max < attackRatio) {
42+
// fast attack
43+
return t / max / attackRatio;
44+
} else {
45+
// slow descent (linear)
46+
return (1 - t / max) / (1 - attackRatio);
47+
}
48+
});
49+
50+
for (let i = 0; i < sampleCount; i++) {
51+
samples += toBytes(
52+
((Math.cos((2 * Math.PI * i * freq) / sampling) + 1) / 2) *
53+
volume *
54+
fadeout(i, sampleCount),
55+
2,
56+
);
57+
}
58+
59+
let dataChunk = [
60+
"fmt ",
61+
"\x10\x00\x00\x00", // 16: chunk length
62+
"\x01\x00", // audio format, PCM integer
63+
"\x01\x00", // mono
64+
toBytes(sampling, 4),
65+
toBytes(sampling * 2, 4), // bytes / sec
66+
"\x02\x00", // bytes per bloc
67+
"\x10\x00", // bytes per sample
68+
"data",
69+
toBytes(sampleCount * 2, 4),
70+
samples,
71+
].join("");
72+
73+
let wav = ["RIFF", toBytes(20 + dataChunk.length, 4), "WAVE", dataChunk].join("");
74+
75+
let uint8 = new Uint8Array(wav.length);
76+
for (let i = 0; i < uint8.length; i++) {
77+
uint8[i] = wav.charCodeAt(i);
78+
}
79+
80+
let blob = new Blob([uint8], { type: "audio/wav" });
81+
return URL.createObjectURL(blob);
82+
}

vite.config.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,7 @@ export default defineConfig(({ command, mode }) => {
107107
providerOptions: {
108108
capabilities: {
109109
"goog:chromeOptions": {
110-
args: [
111-
"--autoplay-policy=no-user-gesture-required"
112-
],
110+
args: ["--autoplay-policy=no-user-gesture-required"],
113111
},
114112
},
115113
},

0 commit comments

Comments
 (0)