Skip to content

Commit 3a0cecf

Browse files
Matze99Matze99
committed
debugged non stack silence removal
1 parent 7802d1f commit 3a0cecf

File tree

3 files changed

+159
-32
lines changed

3 files changed

+159
-32
lines changed

‎playground/main.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,17 @@ console.log(silences);
4141

4242
const audioTest = await new core.AudioClip(audioSource, {
4343
volume: 0.1,
44-
});
44+
}).offsetBy(new core.Timestamp(10000));
4545
console.log("duration", audioTest.duration.millis);
4646

4747
await audioTrack.add(audioTest);
48+
await audioTrack.removeSilences();
49+
50+
console.log("clips", audioTrack.clips);
51+
console.log("duration", audioTrack.clips.at(0)?.start);
52+
console.log("duration", audioTrack.clips.at(0)?.stop);
53+
console.log("duration", audioTrack.clips.at(1)?.start);
54+
console.log("duration", audioTrack.clips.at(1)?.stop);
4855

4956
image.animate()
5057
.rotation(-16).to(14, 5).to(-7, 10).to(24, 7).to(-3, 9).to(19, 7).to(-14, 12).to(5, 9).to(-30, 13)

‎src/tracks/media/media.spec.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,19 @@ import { Composition } from '../../composition';
1010
import { MediaClip } from '../../clips';
1111
import { Timestamp } from '../../models';
1212
import { MediaTrack } from './media';
13+
import { AudioSource } from '../../sources';
14+
15+
class MockMediaClip extends MediaClip {
16+
constructor(duration: number, range: [Timestamp, Timestamp], silences: { start: Timestamp, stop: Timestamp }[], element: HTMLMediaElement) {
17+
super();
18+
this.duration.millis = range[1].millis - range[0].millis;
19+
this.range = range;
20+
this.source = {
21+
silences: async () => silences,
22+
} as any as AudioSource;
23+
this.element = element;
24+
}
25+
}
1326

1427
describe('The Media Track Object', () => {
1528
let comp: Composition;
@@ -23,6 +36,83 @@ describe('The Media Track Object', () => {
2336
track.on('update', updateMock);
2437
});
2538

39+
it('ignores no silences', async () => {
40+
const clip = new MediaClip();
41+
clip.duration.frames = 30;
42+
await track.add(clip);
43+
await track.removeSilences();
44+
expect(track.clips.length).toBe(1);
45+
});
46+
47+
// it('ignores not applicable silences', async () => {
48+
// const clip = new MockMediaClip(30000, [new Timestamp(10000), new Timestamp(20000)], [
49+
// {
50+
// start: new Timestamp(0),
51+
// stop: new Timestamp(500),
52+
// },
53+
// {
54+
// start: new Timestamp(30000),
55+
// stop: new Timestamp(30500),
56+
// },
57+
// ], new Audio());
58+
// await track.add(clip);
59+
// expect(clip.source).toBeDefined();
60+
// await track.removeSilences();
61+
// expect(track.clips.length).toBe(1);
62+
// expect(track.clips.at(0)).toBe(clip);
63+
// });
64+
65+
it('removes silences', async () => {
66+
const clip = new MockMediaClip(30000, [new Timestamp(10000), new Timestamp(20000)], [
67+
{
68+
start: new Timestamp(0),
69+
stop: new Timestamp(10050),
70+
},
71+
{
72+
start: new Timestamp(11000),
73+
stop: new Timestamp(15000),
74+
},
75+
{
76+
start: new Timestamp(19000),
77+
stop: new Timestamp(30500),
78+
},
79+
], new Audio());
80+
await track.add(clip);
81+
expect(clip.source).toBeDefined();
82+
await track.removeSilences();
83+
expect(track.clips.length).toBe(2);
84+
expect(track.clips.at(0)?.range[0].millis).toBe(10051);
85+
expect(track.clips.at(0)?.range[1].millis).toBe(11000);
86+
expect(track.clips.at(1)?.range[0].millis).toBe(15001);
87+
expect(track.clips.at(1)?.range[1].millis).toBe(19000);
88+
});
89+
90+
it('removes silences stacked', async () => {
91+
track.stacked(true);
92+
const clip = new MockMediaClip(30000, [new Timestamp(10000), new Timestamp(20000)], [
93+
{
94+
start: new Timestamp(0),
95+
stop: new Timestamp(10050),
96+
},
97+
{
98+
start: new Timestamp(11000),
99+
stop: new Timestamp(15000),
100+
},
101+
{
102+
start: new Timestamp(19000),
103+
stop: new Timestamp(30500),
104+
},
105+
], new Audio());
106+
await track.add(clip);
107+
expect(clip.source).toBeDefined();
108+
await track.removeSilences();
109+
expect(track.clips.length).toBe(2);
110+
expect(track.clips.at(0)?.range[0].millis).toBe(10051);
111+
expect(track.clips.at(0)?.range[1].millis).toBe(11000);
112+
expect(track.clips.at(1)?.range[0].millis).toBe(11001);
113+
expect(track.clips.at(1)?.range[1].millis).toBe(15000);
114+
});
115+
26116
it('should propagate a seek call', async () => {
27117
const clip = new MediaClip();
28118
clip.element = new Audio();

‎src/tracks/media/media.ts‎

Lines changed: 61 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
/**
22
* Copyright (c) 2024 The Diffusion Studio Authors
33
*
4-
* This Source Code Form is subject to the terms of the Mozilla
4+
* This Source Code Form is subject to the terms of the Mozilla
55
* Public License, v. 2.0 that can be found in the LICENSE file.
66
*/
77

88
import { Track } from '../track';
99

1010
import type { MediaClip } from '../../clips';
11-
import type { Timestamp } from '../../models';
11+
import { Timestamp } from '../../models';
12+
import { StackInsertStrategy } from '../track/track.strategies';
1213

1314
export class MediaTrack<Clip extends MediaClip> extends Track<MediaClip> {
1415
public clips: Clip[] = [];
@@ -19,32 +20,61 @@ export class MediaTrack<Clip extends MediaClip> extends Track<MediaClip> {
1920
}
2021

2122
/**
22-
* Detect periods of silence across all clips in the track
23-
*
24-
* This currently only searches for silences in each clip individually
25-
*
26-
* @param subSample Number of samples to skip when analyzing audio (higher = faster but less accurate)
27-
* @param silenceThreshold Volume threshold in dB below which is considered silence
28-
* @param minSilenceDuration Minimum duration in seconds for a silence period to be included
29-
* @returns Array of silence periods with start and stop times in seconds
30-
*/
31-
public async removeSilences(
32-
subSample: number = 1000,
33-
silenceThreshold: number = -10,
34-
minSilenceDuration: number = 0.1
35-
) {
36-
37-
// Process each clip
38-
for (const clip of this.clips) {
39-
if (!clip.element) {
40-
continue;
41-
}
42-
43-
const silences = await clip.source.silences({});
44-
if (silences.length === 0) continue;
45-
46-
47-
48-
}
49-
}
50-
}
23+
* Detect periods of silence across all clips in the track
24+
*
25+
* This currently only searches for silences in each clip individually
26+
*
27+
* @returns Array of silence periods with start and stop times in seconds
28+
*/
29+
public async removeSilences() {
30+
// if (!(this.strategy instanceof StackInsertStrategy)) {
31+
// throw new Error("Cannot remove silences from a non-stacked track");
32+
// }
33+
34+
// Process each clip
35+
for (const clip of this.clips) {
36+
if (!clip.element) {
37+
continue;
38+
}
39+
40+
const silences = await clip.source.silences({});
41+
if (silences.length === 0) {
42+
continue;
43+
}
44+
45+
const applicableSilences = silences.filter(
46+
(silence) =>
47+
(silence.start.millis > clip.range[0].millis &&
48+
silence.start.millis < clip.range[1].millis) ||
49+
(silence.stop.millis < clip.range[1].millis &&
50+
silence.stop.millis > clip.range[0].millis),
51+
);
52+
if (applicableSilences.length === 0) {
53+
continue;
54+
}
55+
56+
let start = clip.range[0];
57+
let currentClip = clip;
58+
59+
for (const silence of applicableSilences) {
60+
if (silence.start.millis < start.millis) {
61+
const newClip = await currentClip.split(silence.stop.add(currentClip.offset));
62+
currentClip.detach();
63+
start = silence.stop;
64+
currentClip = newClip;
65+
continue;
66+
}
67+
68+
if (silence.stop.millis > currentClip.range[1].millis) {
69+
currentClip = await currentClip.split(silence.start.add(currentClip.offset));
70+
currentClip.detach();
71+
continue;
72+
}
73+
74+
const middleClip = await currentClip.split(silence.start.add(currentClip.offset));
75+
currentClip = await middleClip.split(silence.stop.add(middleClip.offset));
76+
middleClip.detach();
77+
}
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)