improve whisper to work on 8 bit and 32bit wav too, also support form data for language

Author: LostRuins

koboldcpp.py

@@ -1901,7 +1901,8 @@ def log_message(self, format, *args):
             super().log_message(format, *args)
         pass
 
-    def extract_b64string_from_file_upload(self, body):
+    def extract_transcribe_from_file_upload(self, body):
+        result = {"file": None, "prompt": None, "language": None}
         try:
             if 'content-type' in self.headers and self.headers['content-type']:
                 boundary = self.headers['content-type'].split("=")[1].encode()
@@ -1914,15 +1915,27 @@ def extract_b64string_from_file_upload(self, body):
                             file_content_start = fpart.find(b'\r\n\r\n') + 4  # Position after headers
                             file_content_end = fpart.rfind(b'\r\n')  # Ending boundary
                             if file_content_start != -1 and file_content_end != -1:
-                                file_data = fpart[file_content_start:file_content_end]
-                                file_data_base64 = base64.b64encode(file_data).decode('utf-8',"ignore")
-                                base64_string = f"data:audio/wav;base64,{file_data_base64}"
-                                return base64_string
-            print("Uploaded file not found.")
-            return None
+                                if "file" in result and result["file"] is None:
+                                    file_data = fpart[file_content_start:file_content_end]
+                                    file_data_base64 = base64.b64encode(file_data).decode('utf-8',"ignore")
+                                    base64_string = f"data:audio/wav;base64,{file_data_base64}"
+                                    result["file"] = base64_string
+
+                        # Check for fields
+                        detected_prompt_field = re.findall(r'Content-Disposition.*name="prompt"\r\n\r\n(.*)\r\n', fpart.decode('utf-8', errors='ignore'))
+                        if detected_prompt_field and len(detected_prompt_field)>0:
+                            result["prompt"] = detected_prompt_field[0].strip()  # Extract and strip whitespace
+
+                        detected_lang_field = re.findall(r'Content-Disposition.*name="language"\r\n\r\n(.*)\r\n', fpart.decode('utf-8', errors='ignore'))
+                        if detected_lang_field and len(detected_lang_field)>0:
+                            result["language"] = detected_lang_field[0].strip()  # Extract and strip whitespace
+
+            if not ("file" in result and result["file"]):
+                print("Uploaded file not found.")
+            return result
         except Exception as e:
             print(f"File Upload Process Error: {e}")
-            return None
+            return result
 
     async def generate_text(self, genparams, api_format, stream_flag):
         global friendlymodelname, chatcompl_adapter, currfinishreason
@@ -2742,9 +2755,14 @@ def do_POST(self):
                 except Exception:
                     genparams = None
                     if is_transcribe: #fallback handling of file uploads
-                        b64wav = self.extract_b64string_from_file_upload(body)
-                        if b64wav:
+                        formdata = self.extract_transcribe_from_file_upload(body)
+                        if "file" in formdata and formdata["file"]:
+                            b64wav = formdata["file"]
                             genparams = {"audio_data":b64wav}
+                            if "prompt" in formdata and formdata["prompt"]:
+                                genparams["prompt"] = formdata["prompt"]
+                            if "language" in formdata and formdata["language"]:
+                                genparams["language"] = formdata["language"]
 
                     if not genparams:
                         utfprint("Body Err: " + str(body))

otherarch/whispercpp/whisper_adapter.cpp

@@ -57,8 +57,8 @@ static bool read_wav(const std::string & b64data, std::vector<float>& pcmf32, st
         return false;
     }
 
-    if (wav.bitsPerSample != 16) {
-        printf("WAV file must be 16-bit\n");
+    if (wav.bitsPerSample != 8 && wav.bitsPerSample != 16 && wav.bitsPerSample != 32) {
+        printf("WAV file must be 8-bit, 16-bit or 32-bit. Detected: %d\n",wav.bitsPerSample);
         drwav_uninit(&wav);
         return false;
     }
@@ -67,7 +67,23 @@ static bool read_wav(const std::string & b64data, std::vector<float>& pcmf32, st
 
     std::vector<int16_t> pcm16;
     pcm16.resize(n*wav.channels);
-    drwav_read_pcm_frames_s16(&wav, n, pcm16.data());
+
+     if (wav.bitsPerSample == 8) {
+        // Handle 8-bit PCM and convert to 16-bit
+        std::vector<uint8_t> pcm8(n * wav.channels);
+        drwav_read_pcm_frames(&wav, n, pcm8.data());
+        drwav_u8_to_s16(pcm16.data(), pcm8.data(), n * wav.channels);
+    } else if (wav.bitsPerSample == 16) {
+        // Handle 16-bit PCM directly
+        drwav_read_pcm_frames_s16(&wav, n, pcm16.data());
+    } else if (wav.bitsPerSample == 32) {
+        // Handle 32-bit PCM and convert to 16-bit
+        std::vector<int32_t> pcm32(n * wav.channels);
+        drwav_read_pcm_frames_s32(&wav, n, pcm32.data());
+        for (uint64_t i = 0; i < n * wav.channels; ++i) {
+            pcm16[i] = static_cast<int16_t>(pcm32[i] >> 16); // Scale down by shifting
+        }
+    }
     drwav_uninit(&wav);
 
     std::vector<float> raw_pcm;