DashScope: added support for qwen3-tts (#608)

* feat: added support for qwen3-tts

* fix: issues in qwen3.5 tool call tests

The dashscope gateway was updated yesterday, implementing stronger validation of input parameters for tool calls, which caused existing test cases in qwen3.5 to fail.

The cause was not identified initially. To avoid impacting qwen3-tts testing, qwen3.5 tool call tests were temporarily suspended.

A fix has now been made, restoring tool call tests for qwen3.5.

Author: jiangsier-xyz

models/langchain4j-community-dashscope/pom.xml

@@ -21,7 +21,7 @@
     </licenses>
 
     <properties>
-        <dashscope.version>2.22.12</dashscope.version>
+        <dashscope.version>2.22.13</dashscope.version>
     </properties>
 
     <dependencies>

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenChatModel.java

@@ -108,6 +108,7 @@ protected QwenChatModel(
                 .enableSearch(getOrDefault(enableSearch, qwenParameters.enableSearch()))
                 .searchOptions(qwenParameters.searchOptions())
                 .asrOptions(qwenParameters.asrOptions())
+                .ttsOptions(qwenParameters.ttsOptions())
                 .translationOptions(qwenParameters.translationOptions())
                 .vlHighResolutionImages(qwenParameters.vlHighResolutionImages())
                 .isMultimodalModel(getOrDefault(isMultimodalModel, qwenParameters.isMultimodalModel()))

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenChatRequestParameters.java

@@ -33,6 +33,11 @@ public class QwenChatRequestParameters extends DefaultChatRequestParameters {
      * See <a href="https://modelstudio.console.alibabacloud.com/ap-southeast-1/?tab=api#/api/?type=model&url=2986952">Qwen-ASR API reference</a> for more details.
      */
     private final AsrOptions asrOptions;
+    /**
+     * Parameters for text-to-speech (TTS).
+     * See <a href="https://www.alibabacloud.com/help/en/model-studio/qwen-tts">Speech synthesis - Qwen</a> for more details.
+     */
+    private final TtsOptions ttsOptions;
     /**
      * The translation parameters you need to configure when you use the translation
      * models.
@@ -122,6 +127,7 @@ protected QwenChatRequestParameters(Builder builder) {
         this.enableSearch = builder.enableSearch;
         this.searchOptions = builder.searchOptions;
         this.asrOptions = builder.asrOptions;
+        this.ttsOptions = builder.ttsOptions;
         this.translationOptions = builder.translationOptions;
         this.vlHighResolutionImages = builder.vlHighResolutionImages;
         this.isMultimodalModel = builder.isMultimodalModel;
@@ -155,6 +161,10 @@ public AsrOptions asrOptions() {
         return asrOptions;
     }
 
+    public TtsOptions ttsOptions() {
+        return ttsOptions;
+    }
+
     public TranslationOptions translationOptions() {
         return translationOptions;
     }
@@ -236,6 +246,7 @@ public boolean equals(Object o) {
                 && Objects.equals(enableSearch, that.enableSearch)
                 && Objects.equals(searchOptions, that.searchOptions)
                 && Objects.equals(asrOptions, that.asrOptions)
+                && Objects.equals(ttsOptions, that.ttsOptions)
                 && Objects.equals(translationOptions, that.translationOptions)
                 && Objects.equals(vlHighResolutionImages, that.vlHighResolutionImages)
                 && Objects.equals(isMultimodalModel, that.isMultimodalModel)
@@ -261,6 +272,7 @@ public int hashCode() {
                 enableSearch,
                 searchOptions,
                 asrOptions,
+                ttsOptions,
                 translationOptions,
                 vlHighResolutionImages,
                 isMultimodalModel,
@@ -295,7 +307,8 @@ public String toString() {
                 + seed + ", enableSearch="
                 + enableSearch + ", searchOptions="
                 + searchOptions + ", asrOptions="
-                + asrOptions + ", translationOptions="
+                + asrOptions + ", ttsOptions="
+                + ttsOptions + ", translationOptions="
                 + translationOptions + ", vlHighResolutionImages="
                 + vlHighResolutionImages + ", isMultimodalModel="
                 + isMultimodalModel + ", supportIncrementalOutput="
@@ -318,6 +331,7 @@ public static class Builder extends DefaultChatRequestParameters.Builder<Builder
         private Boolean enableSearch;
         private SearchOptions searchOptions;
         private AsrOptions asrOptions;
+        private TtsOptions ttsOptions;
         private TranslationOptions translationOptions;
         private Boolean vlHighResolutionImages;
         private Boolean isMultimodalModel;
@@ -342,6 +356,7 @@ public Builder overrideWith(ChatRequestParameters parameters) {
                 enableSearch(getOrDefault(qwenParameters.enableSearch(), enableSearch));
                 searchOptions(getOrDefault(qwenParameters.searchOptions(), searchOptions));
                 asrOptions(getOrDefault(qwenParameters.asrOptions(), asrOptions));
+                ttsOptions(getOrDefault(qwenParameters.ttsOptions(), ttsOptions));
                 translationOptions(getOrDefault(qwenParameters.translationOptions(), translationOptions));
                 vlHighResolutionImages(getOrDefault(qwenParameters.vlHighResolutionImages(), vlHighResolutionImages));
                 enableThinking(getOrDefault(qwenParameters.enableThinking(), enableThinking));
@@ -382,6 +397,11 @@ public Builder asrOptions(AsrOptions asrOptions) {
             return this;
         }
 
+        public Builder ttsOptions(TtsOptions ttsOptions) {
+            this.ttsOptions = ttsOptions;
+            return this;
+        }
+
         public Builder translationOptions(TranslationOptions translationOptions) {
             this.translationOptions = translationOptions;
             return this;
@@ -656,4 +676,55 @@ public AsrOptions build() {
             }
         }
     }
+
+    /**
+     * Text-to-speech (TTS) parameters.
+     *
+     * @param voice                (Required) The voice to use.
+     *                             See <a href="https://www.alibabacloud.com/help/en/model-studio/qwen-tts#bac280ddf5a1u">Supported system voices</a> for details.
+     * @param languageType         Specify the language of the synthesized audio. The default value is Auto.
+     *                             Auto: Use when text language is uncertain or contains multiple languages. The model automatically matches pronunciation for different language segments, but accuracy is not guaranteed.
+     *                             Specify language: Use when the text is in a single language. Specifying the exact language significantly improves synthesis quality and usually outperforms Auto. Supported values include the following (for now): Chinese, English, German, Italian, Portuguese, Spanish, Japanese, Korean, French, Russian
+     * @param instructions         Provide instructions to guide speech synthesis. Only supported by instruction models.
+     * @param optimizeInstructions Optimize instructions to improve speech naturalness and expressiveness. Defaults to false.
+     *                             Behavior: When true, the system semantically enhances and rewrites instructions to generate internal instructions better suited for speech synthesis.
+     *                             Scenarios: Enable for high-quality, fine-grained speech expression.
+     *                             Dependency: Requires instructions parameter. Has no effect if the instructions parameter is empty.
+     */
+    public record TtsOptions(String voice, String languageType, String instructions, Boolean optimizeInstructions) {
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public static class Builder {
+            private String voice;
+            private String languageType;
+            private String instructions;
+            private Boolean optimizeInstructions;
+
+            public Builder voice(String voice) {
+                this.voice = voice;
+                return this;
+            }
+
+            public Builder languageType(String languageType) {
+                this.languageType = languageType;
+                return this;
+            }
+
+            public Builder instructions(String instructions) {
+                this.instructions = instructions;
+                return this;
+            }
+
+            public Builder optimizeInstructions(Boolean optimizeInstructions) {
+                this.optimizeInstructions = optimizeInstructions;
+                return this;
+            }
+
+            public TtsOptions build() {
+                return new TtsOptions(voice, languageType, instructions, optimizeInstructions);
+            }
+        }
+    }
 }

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenHelper.java

@@ -8,6 +8,7 @@
 import static dev.langchain4j.data.message.ChatMessageType.USER;
 import static dev.langchain4j.internal.JsonSchemaElementUtils.toMap;
 import static dev.langchain4j.internal.Utils.getOrDefault;
+import static dev.langchain4j.internal.Utils.isNotNullOrEmpty;
 import static dev.langchain4j.internal.Utils.isNullOrBlank;
 import static dev.langchain4j.internal.Utils.isNullOrEmpty;
 import static dev.langchain4j.model.chat.request.ToolChoice.REQUIRED;
@@ -24,6 +25,7 @@
 import com.alibaba.dashscope.aigc.generation.GenerationResult;
 import com.alibaba.dashscope.aigc.generation.SearchInfo;
 import com.alibaba.dashscope.aigc.generation.TranslationOptions;
+import com.alibaba.dashscope.aigc.multimodalconversation.AudioParameters;
 import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationOutput;
 import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationParam;
 import com.alibaba.dashscope.aigc.multimodalconversation.MultiModalConversationResult;
@@ -85,6 +87,8 @@
 class QwenHelper {
 
     private static final Logger log = LoggerFactory.getLogger(QwenHelper.class);
+    public static final String GENERATED_AUDIOS_KEY =
+            "generated_audios"; // key for storing generated audios in AiMessage attributes
 
     static List<Message> toQwenMessages(List<ChatMessage> messages, Boolean enableSanitizeMessages) {
         List<ChatMessage> inputMessages =
@@ -168,7 +172,10 @@ static List<Map<String, Object>> toMultiModalContents(ChatMessage message) {
                         .contents().stream()
                                 .map(QwenHelper::toMultiModalContent)
                                 .collect(toList());
-            case AI -> Collections.singletonList(Collections.singletonMap("text", ((AiMessage) message).text()));
+            case AI ->
+                isNullOrBlank(((AiMessage) message).text())
+                        ? Collections.emptyList()
+                        : Collections.singletonList(Collections.singletonMap("text", ((AiMessage) message).text()));
             case SYSTEM ->
                 Collections.singletonList(Collections.singletonMap("text", ((SystemMessage) message).text()));
             case TOOL_EXECUTION_RESULT ->
@@ -335,6 +342,24 @@ static List<Image> imagesFrom(MultiModalConversationResult result) {
                 .collect(toList());
     }
 
+    static List<Audio> audiosFrom(MultiModalConversationResult result) {
+        if (result.getOutput().getAudio() != null) {
+            if (result.getOutput().getAudio().getUrl() != null) {
+                return Collections.singletonList(Audio.builder()
+                        .url(result.getOutput().getAudio().getUrl())
+                        .mimeType("audio/wav")
+                        .build());
+            } else if (result.getOutput().getAudio().getData() != null) {
+                // The base64-encoded audio would be returned in the streaming mode.
+                return Collections.singletonList(Audio.builder()
+                        .base64Data(result.getOutput().getAudio().getData())
+                        .mimeType("audio/pcm")
+                        .build());
+            }
+        }
+        return Collections.emptyList();
+    }
+
     static TokenUsage tokenUsageFrom(GenerationResult result) {
         return Optional.of(result)
                 .map(GenerationResult::getUsage)
@@ -366,11 +391,16 @@ static FinishReason finishReasonFrom(GenerationResult result) {
     }
 
     static FinishReason finishReasonFrom(MultiModalConversationResult result) {
-        MultiModalConversationOutput.Choice choice =
-                result.getOutput().getChoices().get(0);
-        // Upon observation, when tool_calls occur, the returned finish_reason may be null or "stop", not "tool_calls".
-        String finishReason =
-                isNullOrEmpty(choice.getMessage().getToolCalls()) ? choice.getFinishReason() : "tool_calls";
+        String finishReason;
+        if (isNullOrEmpty(result.getOutput().getChoices())) {
+            finishReason = result.getOutput().getFinishReason();
+        } else {
+            MultiModalConversationOutput.Choice choice =
+                    result.getOutput().getChoices().get(0);
+            // Upon observation, when tool_calls occur, the returned finish_reason may be null or "stop", not
+            // "tool_calls".
+            finishReason = isNullOrEmpty(choice.getMessage().getToolCalls()) ? choice.getFinishReason() : "tool_calls";
+        }
 
         return finishReason == null
                 ? null
@@ -411,7 +441,8 @@ static boolean isMultimodalModelName(String modelName) {
                 || modelName.contains("-omni-")
                 || modelName.contains("-image-")
                 || modelName.startsWith("qwen3.5-")
-                || modelName.contains("-asr-");
+                || modelName.contains("-asr-")
+                || modelName.contains("-tts-");
     }
 
     static boolean isSupportingIncrementalOutputModelName(String modelName) {
@@ -582,10 +613,18 @@ static AiMessage aiMessageFrom(MultiModalConversationResult result) {
         String text = answerFrom(result);
         String reasoningContentFrom = reasoningContentFrom(result);
         List<Image> images = imagesFrom(result);
+        List<Audio> audios = audiosFrom(result);
+        Map<String, Object> attributes = new HashMap<>(2);
+        if (isNotNullOrEmpty(images)) {
+            attributes.put(GENERATED_IMAGES_KEY, images);
+        }
+        if (isNotNullOrEmpty(audios)) {
+            attributes.put(GENERATED_AUDIOS_KEY, audios);
+        }
         AiMessage.Builder aiMessageBuilder = AiMessage.builder()
                 .text(text)
                 .thinking(isNullOrBlank(reasoningContentFrom) ? null : reasoningContentFrom)
-                .attributes(isNullOrEmpty(images) ? Map.of() : Map.of(GENERATED_IMAGES_KEY, images));
+                .attributes(attributes);
         if (isFunctionToolCalls(result)) {
             aiMessageBuilder = aiMessageBuilder.toolExecutionRequests(toolExecutionRequestsFrom(result));
             if (text.isBlank()) {
@@ -845,6 +884,10 @@ static void validateGenerationParameters(QwenChatRequestParameters parameters) {
         if (parameters.asrOptions() != null) {
             throw new UnsupportedFeatureException("asrOptions is not supported by " + parameters.modelName());
         }
+
+        if (parameters.ttsOptions() != null) {
+            throw new UnsupportedFeatureException("ttsOptions is not supported by " + parameters.modelName());
+        }
     }
 
     static void validateMultimodalConversationParameters(QwenChatRequestParameters parameters) {
@@ -977,6 +1020,24 @@ static MultiModalConversationParam toMultiModalConversationParam(
             builder.parameter("asr_options", asrOptions);
         }
 
+        if (parameters.ttsOptions() != null) {
+            builder.text(toQwenTtsText(chatRequest.messages()));
+            builder.voice(toQwenTtsVoice(parameters.ttsOptions().voice()));
+            if (parameters.ttsOptions().languageType() != null) {
+                builder.languageType(parameters.ttsOptions().languageType());
+            }
+            if (parameters.ttsOptions().instructions() != null) {
+                // no java field is provided yet
+                builder.parameter("instructions", parameters.ttsOptions().instructions());
+            }
+            if (parameters.ttsOptions().optimizeInstructions() != null) {
+                // no java field is provided yet
+                builder.parameter(
+                        "optimize_instructions", parameters.ttsOptions().optimizeInstructions());
+            }
+            builder.parameter("enable_omni_output_audio_url", true);
+        }
+
         if (parameters.custom() != null) {
             // no java field is provided yet
             builder.parameter("custom", parameters.custom());
@@ -989,6 +1050,23 @@ static MultiModalConversationParam toMultiModalConversationParam(
         return builder.build();
     }
 
+    static String toQwenTtsText(List<ChatMessage> messages) {
+        try {
+            return ((UserMessage) messages.get(messages.size() - 1)).singleText();
+        } catch (Exception e) {
+            throw new IllegalArgumentException("No valid text found", e);
+        }
+    }
+
+    static AudioParameters.Voice toQwenTtsVoice(String voice) {
+        for (AudioParameters.Voice qwenVoice : AudioParameters.Voice.values()) {
+            if (qwenVoice.getValue().equalsIgnoreCase(voice)) {
+                return qwenVoice;
+            }
+        }
+        throw new IllegalArgumentException("Invalid voice: " + voice);
+    }
+
     static com.alibaba.dashscope.common.ResponseFormat toQwenResponseFormat(
             ResponseFormat responseFormat, Boolean jsonSchemaStrict) {
         if (responseFormat == null) {

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenModelName.java

@@ -79,6 +79,9 @@ public class QwenModelName {
             "qwen-audio-turbo-latest"; // Qwen audio understanding model, latest version
 
     public static final String QWEN3_ASR_FLASH = "qwen3-asr-flash"; // Qwen3 ASR model (flash)
+    public static final String QWEN3_TTS_FLASH = "qwen3-tts-flash"; // Qwen3 TTS model (flash)
+    public static final String QWEN3_TTS_INSTRUCT_FLASH =
+            "qwen3-tts-instruct-flash"; // Qwen3 TTS instruction model (flash)
     public static final String QWEN_MT_TURBO = "qwen-mt-turbo"; // Qwen turbo model for translation (deprecated)
     public static final String QWEN_MT_PLUS = "qwen-mt-plus"; // Qwen plus model for translation
     public static final String QWEN_MT_FLASH = "qwen-mt-flash"; // Qwen flash model (2b) for translation

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenStreamingChatModel.java

@@ -123,6 +123,7 @@ public QwenStreamingChatModel(
                 .enableSearch(getOrDefault(enableSearch, qwenParameters.enableSearch()))
                 .searchOptions(qwenParameters.searchOptions())
                 .asrOptions(qwenParameters.asrOptions())
+                .ttsOptions(qwenParameters.ttsOptions())
                 .translationOptions(qwenParameters.translationOptions())
                 .vlHighResolutionImages(qwenParameters.vlHighResolutionImages())
                 .isMultimodalModel(getOrDefault(isMultimodalModel, qwenParameters.isMultimodalModel()))

models/langchain4j-community-dashscope/src/main/java/dev/langchain4j/community/model/dashscope/QwenStreamingResponseBuilder.java

@@ -138,6 +138,10 @@ public QwenPartialResponse append(MultiModalConversationResult partialResponse)
                 String generatedContent = answerFrom(accumulatedMultiModalConversationResult);
                 partialContent = partialContent.substring(generatedContent.length());
             }
+        } else if (partialResponse.getOutput().getAudio() != null
+                && partialResponse.getOutput().getAudio().getData() != null) {
+            // The tts models will incrementally return base64-encoded PCM data.
+            partialContent = partialResponse.getOutput().getAudio().getData();
         }
         if (hasReasoningContent(partialResponse)) {
             partialReasoningContent = reasoningContentFrom(partialResponse);
@@ -657,13 +661,6 @@ private static String merge(String previous, String current) {
     }
 
     private static Long merge(Long previous, Long current) {
-        if (previous == null) {
-            return current;
-        }
-        if (current == null) {
-            return previous;
-        }
-        String resultStr = merge(String.valueOf(previous), String.valueOf(current));
-        return Long.parseLong(resultStr);
+        return current == null ? previous : current;
     }
 }