try to return the remote audio ID when using the chat endpoint with a multimodal model

Signed-off-by: Julien Veyssier <julien-nc@posteo.net>

Author: julien-nc

lib/TaskProcessing/AudioToAudioChatProvider.php

@@ -127,7 +127,13 @@ public function getOutputShapeEnumValues(): array {
 	}
 
 	public function getOptionalOutputShape(): array {
-		return [];
+		return [
+			'audio_id' => new ShapeDescriptor(
+				$this->l->t('Remote audio ID'),
+				$this->l->t('The ID of the audio response returned by the remote service'),
+				EShapeType::Text
+			),
+		];
 	}
 
 	public function getOptionalOutputShapeEnumValues(): array {
@@ -187,58 +193,21 @@ public function process(?string $userId, array $input, callable $reportProgress)
 		$sttModel = $this->appConfig->getValueString(Application::APP_ID, 'default_stt_model_id', Application::DEFAULT_MODEL_ID) ?: Application::DEFAULT_MODEL_ID;
 		$serviceName = $this->appConfig->getValueString(Application::APP_ID, 'service_name') ?: Application::APP_ID;
 
-		/////////////// Using the chat API if connected to OpenAI
+		// Using the chat API if connected to OpenAI
 		// there is an issue if the history mostly contains text, the model will answer text even if we add the audio modality
-		/*
 		if ($this->openAiAPIService->isUsingOpenAi()) {
 			return $this->oneStep($userId, $systemPrompt, $inputFile, $history, $outputVoice, $sttModel, $llmModel, $ttsModel, $speed, $serviceName);
 		}
-		*/
-
-		//////////////// 3 steps: STT -> LLM -> TTS
-		// speech to text
-		try {
-			$inputTranscription = $this->openAiAPIService->transcribeFile($userId, $inputFile, false, $sttModel);
-		} catch (Exception $e) {
-			$this->logger->warning($serviceName . ' transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
-			throw new RuntimeException($serviceName . ' transcription failed with: ' . $e->getMessage());
-		}
-
-		// free prompt
-		try {
-			$completion = $this->openAiAPIService->createChatCompletion($userId, $llmModel, $inputTranscription, $systemPrompt, $history, 1, 1000);
-			$completion = $completion['messages'];
-		} catch (Exception $e) {
-			throw new RuntimeException($serviceName . ' chat completion request failed: ' . $e->getMessage());
-		}
-		if (count($completion) === 0) {
-			throw new RuntimeException('No completion in ' . $serviceName . ' response.');
-		}
-		$llmResult = array_pop($completion);
-
-		// text to speech
-		try {
-			$apiResponse = $this->openAiAPIService->requestSpeechCreation($userId, $llmResult, $ttsModel, $outputVoice, $speed);
 
-			if (!isset($apiResponse['body'])) {
-				$this->logger->warning($serviceName . ' text to speech generation failed: no speech returned');
-				throw new RuntimeException($serviceName . ' text to speech generation failed: no speech returned');
-			}
-			return [
-				'output' => $apiResponse['body'],
-				'output_transcript' => $llmResult,
-				'input_transcript' => $inputTranscription,
-			];
-		} catch (\Exception $e) {
-			$this->logger->warning($serviceName . ' text to speech generation failed with: ' . $e->getMessage(), ['exception' => $e]);
-			throw new RuntimeException($serviceName . ' text to speech generation failed with: ' . $e->getMessage());
-		}
+		// 3 steps: STT -> LLM -> TTS
+		return $this->threeSteps($userId, $systemPrompt, $inputFile, $history, $outputVoice, $sttModel, $llmModel, $ttsModel, $speed, $serviceName);
 	}
 
 	private function oneStep(
 		?string $userId, string $systemPrompt, File $inputFile, array $history, string $outputVoice,
-		string $sttModel, string $llmModel, string $ttsModel, float $speed, string $serviceName
+		string $sttModel, string $llmModel, string $ttsModel, float $speed, string $serviceName,
 	): array {
+		$result = [];
 		$b64Audio = base64_encode($inputFile->getContent());
 		$extraParams = [
 			'modalities' => ['text', 'audio'],
@@ -269,11 +238,12 @@ private function oneStep(
 		} else {
 			$output = base64_decode($message['audio']['data']);
 			$textResponse = $message['audio']['transcript'];
+			if (isset($message['audio']['id'])) {
+				$result['audio_id'] = $message['audio']['id'];
+			}
 		}
-		$result = [
-			'output' => $output,
-			'output_transcript' => $textResponse,
-		];
+		$result['output'] = $output;
+		$result['output_transcript'] = $textResponse;
 
 		// we still want the input transcription
 		try {
@@ -286,4 +256,47 @@ private function oneStep(
 
 		return $result;
 	}
+
+	private function threeSteps(
+		?string $userId, string $systemPrompt, File $inputFile, array $history, string $outputVoice,
+		string $sttModel, string $llmModel, string $ttsModel, float $speed, string $serviceName,
+	): array {
+		// speech to text
+		try {
+			$inputTranscription = $this->openAiAPIService->transcribeFile($userId, $inputFile, false, $sttModel);
+		} catch (Exception $e) {
+			$this->logger->warning($serviceName . ' transcription failed with: ' . $e->getMessage(), ['exception' => $e]);
+			throw new RuntimeException($serviceName . ' transcription failed with: ' . $e->getMessage());
+		}
+
+		// free prompt
+		try {
+			$completion = $this->openAiAPIService->createChatCompletion($userId, $llmModel, $inputTranscription, $systemPrompt, $history, 1, 1000);
+			$completion = $completion['messages'];
+		} catch (Exception $e) {
+			throw new RuntimeException($serviceName . ' chat completion request failed: ' . $e->getMessage());
+		}
+		if (count($completion) === 0) {
+			throw new RuntimeException('No completion in ' . $serviceName . ' response.');
+		}
+		$llmResult = array_pop($completion);
+
+		// text to speech
+		try {
+			$apiResponse = $this->openAiAPIService->requestSpeechCreation($userId, $llmResult, $ttsModel, $outputVoice, $speed);
+
+			if (!isset($apiResponse['body'])) {
+				$this->logger->warning($serviceName . ' text to speech generation failed: no speech returned');
+				throw new RuntimeException($serviceName . ' text to speech generation failed: no speech returned');
+			}
+			return [
+				'output' => $apiResponse['body'],
+				'output_transcript' => $llmResult,
+				'input_transcript' => $inputTranscription,
+			];
+		} catch (\Exception $e) {
+			$this->logger->warning($serviceName . ' text to speech generation failed with: ' . $e->getMessage(), ['exception' => $e]);
+			throw new RuntimeException($serviceName . ' text to speech generation failed with: ' . $e->getMessage());
+		}
+	}
 }

psalm.xml

@@ -39,6 +39,7 @@
 				<referencedClass name="OCP\TaskProcessing\EShapeType" />
 				<referencedClass name="OCP\TaskProcessing\TaskTypes\TextToTextProofread" />
 				<referencedClass name="OCP\TaskProcessing\TaskTypes\TextToTextChatWithTools" />
+				<referencedClass name="OCP\TaskProcessing\TaskTypes\AudioToAudioChat" />
 			</errorLevel>
 		</UndefinedClass>
 		<UndefinedDocblockClass>