Whisper keeps on transcribing Korean audio file into English text.

[cyc9805]

I tried to train Whisper with my custom dataset, which consists of only Korean audio files and texts.
I wrote my code by following the tutorial that is posted on huggingface blog.
However, whenever I inspect evaluation WER, it keeps showing 100.
With an attempt to resolve this problem, I discovered that the model recognizes audio file as English and keeps on transcribing audio files into English text.
I tried every way to make the model transcribe text in Korean, but I had no success.

Could someone please help me with this problem?

[phineas-pta]

can ur model + finetune code be publicly available to be examined ?

[cyc9805]

I did temporally solve this problem by forcing the model to decode language token to korean,
but I'm not sure if it's the best way to solve this problem.

import nlptutti as metrics 
import torch
import evaluate
import numpy as np
from typing import Any, Dict, List, Union
from utils import DataCollatorSpeechSeq2SeqWithPadding, whisper_download
from datasets import load_dataset, load_from_disk
from transformers import WhisperFeatureExtractor, WhisperProcessor, WhisperTokenizer, WhisperForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer
from whisper.normalizers.basic import BasicTextNormalizer


def train(load_from_hub = False):

    rep_name = "seastar105/whisper-medium-ko-zeroth"
    language = 'korean'
    task = 'transcribe'

    model = whisper_download(rep_name)
    tokenizer = WhisperTokenizer.from_pretrained(rep_name, language=language, task=task)
    feature_extractor = WhisperFeatureExtractor.from_pretrained(rep_name)
    processor = WhisperProcessor.from_pretrained("openai/whisper-small", language=language, task=task)

    # Force model to decode token to <|startoftranscript|><|ko|><|transcribe|><|notimestamps|>
    model.config.forced_decoder_ids = [[0, 50258], [1, 50264], [2, 50359], [3, 50363]]

    text_normalizer = BasicTextNormalizer()

    def compute_metrics(pred):

        pred_ids = pred.predictions
        label_ids = pred.label_ids

        # replace -100 with the pad_token_id
        label_ids[label_ids == -100] = tokenizer.pad_token_id

        # we do not want to group tokens when computing the metrics
        pred_str_list = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
        label_str_list = tokenizer.batch_decode(label_ids, skip_special_tokens=True)

        wer_list = []
        cer_list = []

        for pred_str, label_str in zip(pred_str_list, label_str_list):

            pred_str = text_normalizer(pred_str)
            label_str = text_normalizer(label_str)

            cm_wer = metrics.get_wer(pred_str, label_str)['wer']*100
            cm_cer = metrics.get_cer(pred_str, label_str)['cer']*100

            wer_list.append(cm_wer)
            cer_list.append(cm_cer)
        
        avg_wer = np.average(wer_list)
        avg_cer = np.average(cer_list)

        return {"wer": avg_wer, "cer": avg_cer}


    def prepare_dataset(batch):
        # load and resample audio data from 48 to 16kHz
        audio = batch["audio"]

        # compute log-Mel input features from input audio array 
        batch["input_features"] = feature_extractor(audio["array"], sampling_rate=audio["sampling_rate"]).input_features[0]

        # encode target text to label ids 
        if load_from_hub:
            batch["labels"] = tokenizer(batch["text"]).input_ids
        else:
            batch["labels"] = tokenizer(batch["sentence"]).input_ids

        return batch


    if load_from_hub:
        dataset = load_dataset('Bingsu/zeroth-korean')
    else:
        dataset = load_from_disk('/home/cyc/extract_audio/data/table/training')

    dataset = dataset.map(prepare_dataset, remove_columns=dataset.column_names["train"], num_proc=4)

    print(dataset)
    train_dataset = dataset['train']
    eval_dataset = dataset['test']

    data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)

    training_args = Seq2SeqTrainingArguments(
                                            output_dir=f"./model/{rep_name}",  # change to a repo name of your choice
                                            per_device_train_batch_size=8,
                                            gradient_accumulation_steps=2,  # increase by 2x for every 2x decrease in batch size
                                            learning_rate=1e-5,
                                            warmup_steps=500,
                                            max_steps=4000,
                                            gradient_checkpointing=True,
                                            fp16=True,
                                            evaluation_strategy="steps",
                                            per_device_eval_batch_size=8,
                                            predict_with_generate=True,
                                            generation_max_length=400,
                                            save_steps=1000,
                                            eval_steps=100,
                                            logging_steps=25,
                                            report_to=["tensorboard"],
                                            load_best_model_at_end=True,
                                            metric_for_best_model="cer",
                                            greater_is_better=False,
                                            push_to_hub=False
                                            )

    trainer = Seq2SeqTrainer(
                            args=training_args,
                            model=model,
                            train_dataset=train_dataset,
                            eval_dataset=eval_dataset,
                            data_collator=data_collator,
                            compute_metrics=compute_metrics,
                            tokenizer=processor.feature_extractor
                            )
    
    trainer.train()


if __name__ == '__main__':
    train()