python example fails on GPU: type mismatch of torch.cuda.HalfTensor and torch.cuda.FloatTensor

[DSLituiev]

I am trying to run the example from the README.md in a GPU enabled notebook:

import whisper
model = whisper.load_model("base")
result = model.transcribe(file_path)

and getting this:

/opt/conda/lib/python3.8/site-packages/torch/functional.py:515: UserWarning: The function torch.rfft is deprecated and will be removed in a future PyTorch release. Use the new torch.fft module functions, instead, by importing torch.fft and calling torch.fft.fft or torch.fft.rfft. (Triggered internally at  /opt/conda/conda-bld/pytorch_1603729096996/work/aten/src/ATen/native/SpectralOps.cpp:590.)
  return _VF.stft(input, n_fft, hop_length, win_length, window,  # type: ignore
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [11], in <cell line: 2>()
      1 model = whisper.load_model("base")
----> 2 result = model.transcribe(ff.path)

File /opt/conda/lib/python3.8/site-packages/whisper/transcribe.py:94, in transcribe(model, audio, verbose, temperature, compression_ratio_threshold, logprob_threshold, no_speech_threshold, condition_on_previous_text, initial_prompt, **decode_options)
     92     print("Detecting language using up to the first 30 seconds. Use `--language` to specify the language")
     93 segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)
---> 94 _, probs = model.detect_language(segment)
     95 decode_options["language"] = max(probs, key=probs.get)
     96 if verbose is not None:

File /opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py:26, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
     23 @functools.wraps(func)
     24 def decorate_context(*args, **kwargs):
     25     with self.__class__():
---> 26         return func(*args, **kwargs)

File /opt/conda/lib/python3.8/site-packages/whisper/decoding.py:43, in detect_language(model, mel, tokenizer)
     41 # skip encoder forward pass if already-encoded audio features were given
     42 if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state):
---> 43     mel = model.encoder(mel)
     45 # forward pass using a single token, startoftranscript
     46 n_audio = mel.shape[0]

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/whisper/model.py:149, in AudioEncoder.forward(self, x)
    144 def forward(self, x: Tensor):
    145     """
    146     x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
    147         the mel spectrogram of the audio
    148     """
--> 149     x = F.gelu(self.conv1(x))
    150     x = F.gelu(self.conv2(x))
    151     x = x.permute(0, 2, 1)

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py:258, in Conv1d.forward(self, input)
    254 if self.padding_mode != 'zeros':
    255     return F.conv1d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
    256                     self.weight, self.bias, self.stride,
    257                     _single(0), self.dilation, self.groups)
--> 258 return F.conv1d(input, self.weight, self.bias, self.stride,
    259                 self.padding, self.dilation, self.groups)

RuntimeError: Input type (torch.cuda.HalfTensor) and weight type (torch.cuda.FloatTensor) should be the same

System:

whisper.__version__: '20230124'
torch.__version__: '1.7.0'
NVIDIA-SMI 470.141.03   Driver Version: 470.141.03   CUDA Version: 11.4 

[DSLituiev]

I tried calling model=model.float() (same error) and model = model.half(), and the latter returns:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [22], in <cell line: 1>()
----> 1 result = model.transcribe(ff.path)

File /opt/conda/lib/python3.8/site-packages/whisper/transcribe.py:94, in transcribe(model, audio, verbose, temperature, compression_ratio_threshold, logprob_threshold, no_speech_threshold, condition_on_previous_text, initial_prompt, **decode_options)
     92     print("Detecting language using up to the first 30 seconds. Use `--language` to specify the language")
     93 segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)
---> 94 _, probs = model.detect_language(segment)
     95 decode_options["language"] = max(probs, key=probs.get)
     96 if verbose is not None:

File /opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py:26, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
     23 @functools.wraps(func)
     24 def decorate_context(*args, **kwargs):
     25     with self.__class__():
---> 26         return func(*args, **kwargs)

File /opt/conda/lib/python3.8/site-packages/whisper/decoding.py:43, in detect_language(model, mel, tokenizer)
     41 # skip encoder forward pass if already-encoded audio features were given
     42 if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state):
---> 43     mel = model.encoder(mel)
     45 # forward pass using a single token, startoftranscript
     46 n_audio = mel.shape[0]

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/whisper/model.py:157, in AudioEncoder.forward(self, x)
    154 x = (x + self.positional_embedding).to(x.dtype)
    156 for block in self.blocks:
--> 157     x = block(x)
    159 x = self.ln_post(x)
    160 return x

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/whisper/model.py:125, in ResidualAttentionBlock.forward(self, x, xa, mask, kv_cache)
    118 def forward(
    119     self,
    120     x: Tensor,
   (...)
    123     kv_cache: Optional[dict] = None,
    124 ):
--> 125     x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]
    126     if self.cross_attn:
    127         x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/whisper/model.py:31, in LayerNorm.forward(self, x)
     30 def forward(self, x: Tensor) -> Tensor:
---> 31     return super().forward(x.float()).type(x.dtype)

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/normalization.py:169, in LayerNorm.forward(self, input)
    168 def forward(self, input: Tensor) -> Tensor:
--> 169     return F.layer_norm(
    170         input, self.normalized_shape, self.weight, self.bias, self.eps)

File /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:2094, in layer_norm(input, normalized_shape, weight, bias, eps)
   2091     if type(input) is not Tensor and has_torch_function((input,)):
   2092         return handle_torch_function(
   2093             layer_norm, (input,), input, normalized_shape, weight=weight, bias=bias, eps=eps)
-> 2094 return torch.layer_norm(input, normalized_shape, weight, bias, eps,
   2095                         torch.backends.cudnn.enabled)

RuntimeError: expected scalar type Float but found Half

[DSLituiev]

I tried reading the audio and casting it to Torch arrays. It still breaks with all combination of model-input of float-float and half-half.
It seems like the model forces the array to half. With half-half:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [38], in <cell line: 1>()
----> 1 result = model.transcribe(torch.from_numpy(data,).half())

File /opt/conda/lib/python3.8/site-packages/whisper/transcribe.py:85, in transcribe(model, audio, verbose, temperature, compression_ratio_threshold, logprob_threshold, no_speech_threshold, condition_on_previous_text, initial_prompt, **decode_options)
     82 if dtype == torch.float32:
     83     decode_options["fp16"] = False
---> 85 mel = log_mel_spectrogram(audio)
     87 if decode_options.get("language", None) is None:
     88     if not model.is_multilingual:

File /opt/conda/lib/python3.8/site-packages/whisper/audio.py:115, in log_mel_spectrogram(audio, n_mels)
    112     audio = torch.from_numpy(audio)
    114 window = torch.hann_window(N_FFT).to(audio.device)
--> 115 stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True)
    116 magnitudes = stft[..., :-1].abs() ** 2
    118 filters = mel_filters(audio.device, n_mels)

File /opt/conda/lib/python3.8/site-packages/torch/functional.py:513, in stft(input, n_fft, hop_length, win_length, window, center, pad_mode, normalized, onesided, return_complex)
    511     extended_shape = [1] * (3 - signal_dim) + list(input.size())
    512     pad = int(n_fft // 2)
--> 513     input = F.pad(input.view(extended_shape), (pad, pad), pad_mode)
    514     input = input.view(input.shape[-signal_dim:])
    515 return _VF.stft(input, n_fft, hop_length, win_length, window,  # type: ignore
    516                 normalized, onesided, return_complex)

File /opt/conda/lib/python3.8/site-packages/torch/nn/functional.py:3559, in _pad(input, pad, mode, value)
   3557 assert len(pad) == 2, '3D tensors expect 2 values for padding'
   3558 if mode == 'reflect':
-> 3559     return torch._C._nn.reflection_pad1d(input, pad)
   3560 elif mode == 'replicate':
   3561     return torch._C._nn.replication_pad1d(input, pad)

RuntimeError: "reflection_pad1d" not implemented for 'Half'

With float-float:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [40], in <cell line: 1>()
----> 1 result = model.transcribe(torch.from_numpy(data,).float())

File /opt/conda/lib/python3.8/site-packages/whisper/transcribe.py:94, in transcribe(model, audio, verbose, temperature, compression_ratio_threshold, logprob_threshold, no_speech_threshold, condition_on_previous_text, initial_prompt, **decode_options)
     92     print("Detecting language using up to the first 30 seconds. Use `--language` to specify the language")
     93 segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)
---> 94 _, probs = model.detect_language(segment)
     95 decode_options["language"] = max(probs, key=probs.get)
     96 if verbose is not None:

File /opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py:26, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
     23 @functools.wraps(func)
     24 def decorate_context(*args, **kwargs):
     25     with self.__class__():
---> 26         return func(*args, **kwargs)

File /opt/conda/lib/python3.8/site-packages/whisper/decoding.py:43, in detect_language(model, mel, tokenizer)
     41 # skip encoder forward pass if already-encoded audio features were given
     42 if mel.shape[-2:] != (model.dims.n_audio_ctx, model.dims.n_audio_state):
---> 43     mel = model.encoder(mel)
     45 # forward pass using a single token, startoftranscript
     46 n_audio = mel.shape[0]

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/whisper/model.py:149, in AudioEncoder.forward(self, x)
    144 def forward(self, x: Tensor):
    145     """
    146     x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
    147         the mel spectrogram of the audio
    148     """
--> 149     x = F.gelu(self.conv1(x))
    150     x = F.gelu(self.conv2(x))
    151     x = x.permute(0, 2, 1)

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py:727, in Module._call_impl(self, *input, **kwargs)
    725     result = self._slow_forward(*input, **kwargs)
    726 else:
--> 727     result = self.forward(*input, **kwargs)
    728 for hook in itertools.chain(
    729         _global_forward_hooks.values(),
    730         self._forward_hooks.values()):
    731     hook_result = hook(self, input, result)

File /opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py:258, in Conv1d.forward(self, input)
    254 if self.padding_mode != 'zeros':
    255     return F.conv1d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
    256                     self.weight, self.bias, self.stride,
    257                     _single(0), self.dilation, self.groups)
--> 258 return F.conv1d(input, self.weight, self.bias, self.stride,
    259                 self.padding, self.dilation, self.groups)

RuntimeError: Input type (torch.cuda.HalfTensor) and weight type (torch.cuda.FloatTensor) should be the same

[DSLituiev]

Following solved it:

result = model.transcribe(torch.from_numpy(data,).float(), language = "en", fp16=False)

[DSLituiev]

May be a good idea to force to float if the input is float

[Wentao795]

hi，i get a good result : result = model.transcribe("xxx.wav",fp16=False)

[JiweiZh]

same envs, result = model.transcribe("xxx.wav",fp16=False) works