Skip to content

Commit a189bb9

Browse files
sanchit-gandhisanchit-gandhi
authored
update trmfs scripts (#37)
1 parent ac455ba commit a189bb9

File tree

6 files changed

+146
-58
lines changed

6 files changed

+146
-58
lines changed

‎transformers/run_data2vec.sh‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,20 @@ do
1414
python run_eval.py \
1515
--model_id=${MODEL_ID} \
1616
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17-
--dataset="ami" \
17+
--dataset="voxpopuli" \
1818
--split="test" \
1919
--device=0 \
2020
--batch_size=${BATCH_SIZE} \
21-
--max_eval_samples=8
21+
--max_eval_samples=-1
2222

23+
python run_eval.py \
24+
--model_id=${MODEL_ID} \
25+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
26+
--dataset="ami" \
27+
--split="test" \
28+
--device=0 \
29+
--batch_size=${BATCH_SIZE} \
30+
--max_eval_samples=-1
2331

2432
python run_eval.py \
2533
--model_id=${MODEL_ID} \
@@ -28,7 +36,7 @@ do
2836
--split="test" \
2937
--device=0 \
3038
--batch_size=${BATCH_SIZE} \
31-
--max_eval_samples=8
39+
--max_eval_samples=-1
3240

3341
python run_eval.py \
3442
--model_id=${MODEL_ID} \
@@ -37,7 +45,7 @@ do
3745
--split="test" \
3846
--device=0 \
3947
--batch_size=${BATCH_SIZE} \
40-
--max_eval_samples=8
48+
--max_eval_samples=-1
4149

4250
python run_eval.py \
4351
--model_id=${MODEL_ID} \
@@ -46,7 +54,7 @@ do
4654
--split="test.clean" \
4755
--device=0 \
4856
--batch_size=${BATCH_SIZE} \
49-
--max_eval_samples=8
57+
--max_eval_samples=-1
5058

5159
python run_eval.py \
5260
--model_id=${MODEL_ID} \
@@ -55,7 +63,7 @@ do
5563
--split="test.other" \
5664
--device=0 \
5765
--batch_size=${BATCH_SIZE} \
58-
--max_eval_samples=8
66+
--max_eval_samples=-1
5967

6068
python run_eval.py \
6169
--model_id=${MODEL_ID} \
@@ -64,7 +72,7 @@ do
6472
--split="test" \
6573
--device=0 \
6674
--batch_size=${BATCH_SIZE} \
67-
--max_eval_samples=8
75+
--max_eval_samples=-1
6876

6977
python run_eval.py \
7078
--model_id=${MODEL_ID} \
@@ -73,16 +81,7 @@ do
7381
--split="test" \
7482
--device=0 \
7583
--batch_size=${BATCH_SIZE} \
76-
--max_eval_samples=8
77-
78-
python run_eval.py \
79-
--model_id=${MODEL_ID} \
80-
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
81-
--dataset="voxpopuli" \
82-
--split="test" \
83-
--device=0 \
84-
--batch_size=${BATCH_SIZE} \
85-
--max_eval_samples=8
84+
--max_eval_samples=-1
8685

8786
# Evaluate results
8887
RUNDIR=`pwd` && \

‎transformers/run_hubert.sh‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,20 @@ do
1414
python run_eval.py \
1515
--model_id=${MODEL_ID} \
1616
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17-
--dataset="ami" \
17+
--dataset="voxpopuli" \
1818
--split="test" \
1919
--device=0 \
2020
--batch_size=${BATCH_SIZE} \
2121
--max_eval_samples=-1
2222

23+
python run_eval.py \
24+
--model_id=${MODEL_ID} \
25+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
26+
--dataset="ami" \
27+
--split="test" \
28+
--device=0 \
29+
--batch_size=${BATCH_SIZE} \
30+
--max_eval_samples=-1
2331

2432
python run_eval.py \
2533
--model_id=${MODEL_ID} \
@@ -75,15 +83,6 @@ do
7583
--batch_size=${BATCH_SIZE} \
7684
--max_eval_samples=-1
7785

78-
python run_eval.py \
79-
--model_id=${MODEL_ID} \
80-
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
81-
--dataset="voxpopuli" \
82-
--split="test" \
83-
--device=0 \
84-
--batch_size=${BATCH_SIZE} \
85-
--max_eval_samples=-1
86-
8786
# Evaluate results
8887
RUNDIR=`pwd` && \
8988
cd ../normalizer && \

‎transformers/run_mms.sh‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,20 @@ do
1414
python run_eval.py \
1515
--model_id=${MODEL_ID} \
1616
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17-
--dataset="ami" \
17+
--dataset="voxpopuli" \
1818
--split="test" \
1919
--device=0 \
2020
--batch_size=${BATCH_SIZE} \
2121
--max_eval_samples=-1
2222

23+
python run_eval.py \
24+
--model_id=${MODEL_ID} \
25+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
26+
--dataset="ami" \
27+
--split="test" \
28+
--device=0 \
29+
--batch_size=${BATCH_SIZE} \
30+
--max_eval_samples=-1
2331

2432
python run_eval.py \
2533
--model_id=${MODEL_ID} \
@@ -75,15 +83,6 @@ do
7583
--batch_size=${BATCH_SIZE} \
7684
--max_eval_samples=-1
7785

78-
python run_eval.py \
79-
--model_id=${MODEL_ID} \
80-
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
81-
--dataset="voxpopuli" \
82-
--split="test" \
83-
--device=0 \
84-
--batch_size=${BATCH_SIZE} \
85-
--max_eval_samples=-1
86-
8786
# Evaluate results
8887
RUNDIR=`pwd` && \
8988
cd ../normalizer && \

‎transformers/run_wav2vec2.sh‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
export PYTHONPATH="..":$PYTHONPATH
44

5-
MODEL_IDs=("facebook/wav2vec2-base-960h" "facebook/wav2vec2-large-960h" "facebook/wav2vec2-large-960h-lv60-self" "facebook/wav2vec2-large-robust-ft-libri-960h" "facebook/wav2vec2-conformer-rel-pos-large-960h-ft" "facebook/wav2vec2-conformer-rope-large-960h-ft")
5+
MODEL_IDs=("facebook/wav2vec2-base-960h" "facebook/wav2vec2-large-960h" "facebook/wav2vec2-large-960h-lv60-self" "facebook/wav2vec2-large-robust-ft-libri-960h")
66
BATCH_SIZE=64
77

88
num_models=${#MODEL_IDs[@]}
@@ -14,12 +14,20 @@ do
1414
python run_eval.py \
1515
--model_id=${MODEL_ID} \
1616
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17-
--dataset="ami" \
17+
--dataset="voxpopuli" \
1818
--split="test" \
1919
--device=0 \
2020
--batch_size=${BATCH_SIZE} \
2121
--max_eval_samples=-1
2222

23+
python run_eval.py \
24+
--model_id=${MODEL_ID} \
25+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
26+
--dataset="ami" \
27+
--split="test" \
28+
--device=0 \
29+
--batch_size=${BATCH_SIZE} \
30+
--max_eval_samples=-1
2331

2432
python run_eval.py \
2533
--model_id=${MODEL_ID} \
@@ -75,15 +83,6 @@ do
7583
--batch_size=${BATCH_SIZE} \
7684
--max_eval_samples=-1
7785

78-
python run_eval.py \
79-
--model_id=${MODEL_ID} \
80-
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
81-
--dataset="voxpopuli" \
82-
--split="test" \
83-
--device=0 \
84-
--batch_size=${BATCH_SIZE} \
85-
--max_eval_samples=-1
86-
8786
# Evaluate results
8887
RUNDIR=`pwd` && \
8988
cd ../normalizer && \

‎transformers/run_wav2vec2_conformer.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/bin/bash
2+
3+
export PYTHONPATH="..":$PYTHONPATH
4+
5+
MODEL_IDs=("facebook/wav2vec2-conformer-rel-pos-large-960h-ft" "facebook/wav2vec2-conformer-rope-large-960h-ft")
6+
BATCH_SIZE=32
7+
8+
num_models=${#MODEL_IDs[@]}
9+
10+
for (( i=0; i<${num_models}; i++ ));
11+
do
12+
MODEL_ID=${MODEL_IDs[$i]}
13+
14+
python run_eval.py \
15+
--model_id=${MODEL_ID} \
16+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17+
--dataset="voxpopuli" \
18+
--split="test" \
19+
--device=0 \
20+
--batch_size=${BATCH_SIZE} \
21+
--max_eval_samples=-1
22+
23+
python run_eval.py \
24+
--model_id=${MODEL_ID} \
25+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
26+
--dataset="ami" \
27+
--split="test" \
28+
--device=0 \
29+
--batch_size=${BATCH_SIZE} \
30+
--max_eval_samples=-1
31+
32+
python run_eval.py \
33+
--model_id=${MODEL_ID} \
34+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
35+
--dataset="earnings22" \
36+
--split="test" \
37+
--device=0 \
38+
--batch_size=${BATCH_SIZE} \
39+
--max_eval_samples=-1
40+
41+
python run_eval.py \
42+
--model_id=${MODEL_ID} \
43+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
44+
--dataset="gigaspeech" \
45+
--split="test" \
46+
--device=0 \
47+
--batch_size=${BATCH_SIZE} \
48+
--max_eval_samples=-1
49+
50+
python run_eval.py \
51+
--model_id=${MODEL_ID} \
52+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
53+
--dataset="librispeech" \
54+
--split="test.clean" \
55+
--device=0 \
56+
--batch_size=${BATCH_SIZE} \
57+
--max_eval_samples=-1
58+
59+
python run_eval.py \
60+
--model_id=${MODEL_ID} \
61+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
62+
--dataset="librispeech" \
63+
--split="test.other" \
64+
--device=0 \
65+
--batch_size=${BATCH_SIZE} \
66+
--max_eval_samples=-1
67+
68+
python run_eval.py \
69+
--model_id=${MODEL_ID} \
70+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
71+
--dataset="spgispeech" \
72+
--split="test" \
73+
--device=0 \
74+
--batch_size=${BATCH_SIZE} \
75+
--max_eval_samples=-1
76+
77+
python run_eval.py \
78+
--model_id=${MODEL_ID} \
79+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
80+
--dataset="tedlium" \
81+
--split="test" \
82+
--device=0 \
83+
--batch_size=${BATCH_SIZE} \
84+
--max_eval_samples=-1
85+
86+
# Evaluate results
87+
RUNDIR=`pwd` && \
88+
cd ../normalizer && \
89+
python -c "import eval_utils; eval_utils.score_results('${RUNDIR}/results', '${MODEL_ID}')" && \
90+
cd $RUNDIR
91+
92+
done

‎transformers/run_whisper.sh‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
export PYTHONPATH="..":$PYTHONPATH
44

5-
MODEL_IDs=("openai/whisper-tiny.en" "openai/whisper-small.en" "openai/whisper-base.en" "openai/whisper-medium.en" "openai/whisper-large" "openai/whisper-large-v2")
5+
MODEL_IDs=("openai/whisper-tiny.en" "openai/whisper-small.en" "openai/whisper-base.en" "openai/whisper-medium.en" "openai/whisper-large" "openai/whisper-large-v2" "openai/whisper-large-v3" "distil-whisper/distil-medium.en" "distil-whisper/distil-large-v2" "distil-whisper/distil-large-v3")
66
BATCH_SIZE=64
77

88
num_models=${#MODEL_IDs[@]}
@@ -11,6 +11,15 @@ for (( i=0; i<${num_models}; i++ ));
1111
do
1212
MODEL_ID=${MODEL_IDs[$i]}
1313

14+
python run_eval.py \
15+
--model_id=${MODEL_ID} \
16+
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
17+
--dataset="voxpopuli" \
18+
--split="test" \
19+
--device=0 \
20+
--batch_size=${BATCH_SIZE} \
21+
--max_eval_samples=-1
22+
1423
python run_eval.py \
1524
--model_id=${MODEL_ID} \
1625
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
@@ -74,15 +83,6 @@ do
7483
--batch_size=${BATCH_SIZE} \
7584
--max_eval_samples=-1
7685

77-
python run_eval.py \
78-
--model_id=${MODEL_ID} \
79-
--dataset_path="hf-audio/esb-datasets-test-only-sorted" \
80-
--dataset="voxpopuli" \
81-
--split="test" \
82-
--device=0 \
83-
--batch_size=${BATCH_SIZE} \
84-
--max_eval_samples=-1
85-
8686
# Evaluate results
8787
RUNDIR=`pwd` && \
8888
cd ../normalizer && \

0 commit comments

Comments
 (0)