+ * - true: Enable + *
+ * - false: Disable (default) + *
+ * Enabling MLLM automatically disables ASR, LLM, and TTS. When you set this + * parameter to true, enable_aivad is also disabled. + *
+ * @since v0.7.0 + */ + @JsonProperty("enable_mllm") + private Boolean enableMLLM; + public static Builder builder() { return new Builder(); } @@ -488,6 +535,262 @@ public AdvancedFeatures build() { } } + + /** + * @brief Multi-modal language model (MLLM) configuration + * @since v0.7.0 + */ + public static class MLLMPayload { + /** + * The WebSocket URL for OpenAI Realtime API(Required) + */ + @JsonProperty("url") + private String url; + + /** + * The API key used for authentication.(Required) + *
+ * Get your API key from the OpenAI Console. + */ + @JsonProperty("api_key") + private String apiKey; + + /** + * Array of conversation items used for short-term memory management.(Optional) + *
+ * Uses the same structure as item.content from the OpenAI Realtime API(https://platform.openai.com/docs/api-reference/realtime-client-events/conversation/item/create).
+ */
+ @JsonProperty("messages")
+ private List
+ * - Modalities override: The modalities setting in params is overridden by `input_modalities` and `output_modalities`.
+ *
+ * - Turn detection override: The turn_detection setting in params is overridden by the `turn_detection` section outside of `mllm`.
+ */
+ @JsonProperty("params")
+ private Map
+ * Cannot exceed the model's context window.
+ *
+ * Default value is 32.
+ */
+ @JsonProperty("max_history")
+ private Integer maxHistory;
+
+ /**
+ * Input modalities for the MLLM (optional)
+ *
+ * - ["audio"]: Audio only (default)
+ *
+ * - ["audio", "text"]: Audio and text.
+ */
+ @JsonProperty("input_modalities")
+ private List
+ * - ["text", "audio"] for both text and voice responses.
+ */
+ @JsonProperty("output_modalities")
+ private List
+ * Set to `openai` for OpenAI Realtime API.
+ */
+ @JsonProperty("vendor")
+ private String vendor;
+
+ /**
+ * API request style.
+ *
+ * Set to `openai` for OpenAI Realtime API format.
+ */
+ @JsonProperty("style")
+ private String style;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private MLLMPayload(Builder builder) {
+ setUrl(builder.url);
+ setApiKey(builder.apiKey);
+ setMessages(builder.messages);
+ setParams(builder.params);
+ setMaxHistory(builder.maxHistory);
+ setInputModalities(builder.inputModalities);
+ setOutputModalities(builder.outputModalities);
+ setGreetingMessage(builder.greetingMessage);
+ setVendor(builder.vendor);
+ setStyle(builder.style);
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getApiKey() {
+ return apiKey;
+ }
+
+ public void setApiKey(String apiKey) {
+ this.apiKey = apiKey;
+ }
+
+ public List
- * Default value is 10
+ * Default value is 32
*
* Passing 0 means no short-term memory is cached. agent and subscribed users
* will record entries separately.
@@ -676,11 +979,11 @@ public void setSystemMessages(List
- * - zh-CN: Chinese (supports mixed Chinese and English) (default)
- *
- * - en-US: English
- */
@JsonProperty("language")
private String language;
+ @JsonProperty("phrase_list")
+ private List
+ * - zh-CN: Chinese (supports mixed Chinese and English) (default)
+ *
+ * - en-US: English
+ */
+ @JsonProperty("language")
+ private String language;
+
+ /**
+ * ASR vendor, see {@link ASRVendorEnum}
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("vendor")
+ private ASRVendorEnum vendor;
+
+ /**
+ * ASR vendor parameter description, see
+ *
+ * - {@link FengmingASRVendorParams}
+ *
+ * - {@link TencentASRVendorParams}
+ *
+ * - {@link MicrosoftASRVendorParams}
+ *
+ * - {@link ARESASRVendorParams}
+ *
+ * - {@link DeepgramASRVendorParams}
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("params")
+ private ASRVendorParams params;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private ASRPayload(Builder builder) {
+ setLanguage(builder.language);
+ setVendor(builder.vendor);
+ setParams(builder.params);
+ }
+
+ public String getLanguage() {
+ return language;
+ }
+
+ public void setLanguage(String language) {
+ this.language = language;
+ }
+
+ public ASRVendorEnum getVendor() {
+ return vendor;
+ }
+
+ public void setVendor(ASRVendorEnum vendor) {
+ this.vendor = vendor;
+ }
+
+ public ASRVendorParams getParams() {
+ return params;
+ }
+
+ public void setParams(ASRVendorParams params) {
+ this.params = params;
+ }
+
public static final class Builder {
private String language;
+ private ASRVendorEnum vendor;
+ private ASRVendorParams params;
private Builder() {
}
@@ -2377,6 +3520,16 @@ public Builder language(String val) {
return this;
}
+ public Builder vendor(ASRVendorEnum val) {
+ vendor = val;
+ return this;
+ }
+
+ public Builder params(ASRVendorParams val) {
+ params = val;
+ return this;
+ }
+
public ASRPayload build() {
return new ASRPayload(this);
}
@@ -2446,16 +3599,213 @@ public void setInterruptMode(String interruptMode) {
@JsonProperty("interrupt_mode")
private String interruptMode;
+ /**
+ * Turn detection mechanism.(Optional)
+ *
+ * - "agora_vad": Agora VAD.(Default)
+ *
+ * - "server_vad": The model detects the start and end of speech based on audio
+ * volume and responds at the end of user speech. Only available when mllm is
+ * enabled and OpenAI is selected.
+ *
+ * - "semantic_vad": Uses a turn detection model in conjunction with VAD to
+ * semantically estimate whether the user has finished speaking, then dynamically
+ * sets a timeout based on this probability for more natural conversations.Only
+ * available when mllm is enabled and OpenAI is selected.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("type")
+ private String type;
+
+ /**
+ * The amount of time in milliseconds that the user's voice must exceed the VAD
+ * threshold before an interruption is triggered.(Optional)
+ *
+ * Default value is 160.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("interrupt_duration_ms")
+ private Integer interruptDurationMs;
+
+ /**
+ * The extra forward padding time in milliseconds before the processing system
+ * starts to process the speech input. This padding helps capture the beginning
+ * of the speech.
+ *
+ * Default value is 800.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("prefix_padding_ms")
+ private Integer prefixPaddingMs;
+
+ /**
+ * The duration of audio silence in milliseconds.(Optional)
+ *
+ * If no voice activity is detected during this period, the agent assumes that
+ * the user has stopped speaking.
+ *
+ * Default value is 480.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("silence_duration_ms")
+ private Integer silenceDurationMs;
+
+ /**
+ * Identification sensitivity determines the level of sound in the audio signal
+ * that is considered voice activity.(Optional)
+ *
+ * Lower values make it easier for the agent to detect speech, and higher values
+ * ignore weak sounds.
+ *
+ * The value range is (0.0, 1.0).
+ *
+ * Default value is 0.5.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("threshold")
+ private Float threshold;
+
+ /**
+ * Whether to automatically generate a response when a VAD stop event occurs.
+ * (Optional)
+ *
+ * Only available in server_vad and semantic_vad modes when using OpenAI
+ * Realtime API.
+ *
+ * Default value is true.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("create_response")
+ private Boolean createResponse;
+
+ /**
+ * Whether to automatically interrupt any ongoing response when a VAD start
+ * event occurs.
+ *
+ * Only available in server_vad and semantic_vad modes when using OpenAI
+ * Realtime API.
+ *
+ * Default value is true.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("interrupt_response")
+ private Boolean interruptResponse;
+
+ /**
+ * The eagerness of the model to respond(Optional):
+ *
+ * - "auto": Equivalent to medium(Default)
+ *
+ * - "low": Wait longer for the user to continue speaking
+ *
+ * - "high": Respond more quickly
+ *
+ * Only available in semantic_vad mode when using OpenAI Realtime API.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("eagerness")
+ private String eagerness;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public Integer getInterruptDurationMs() {
+ return interruptDurationMs;
+ }
+
+ public void setInterruptDurationMs(Integer interruptDurationMs) {
+ this.interruptDurationMs = interruptDurationMs;
+ }
+
+ public Integer getPrefixPaddingMs() {
+ return prefixPaddingMs;
+ }
+
+ public void setPrefixPaddingMs(Integer prefixPaddingMs) {
+ this.prefixPaddingMs = prefixPaddingMs;
+ }
+
+ public Integer getSilenceDurationMs() {
+ return silenceDurationMs;
+ }
+
+ public void setSilenceDurationMs(Integer silenceDurationMs) {
+ this.silenceDurationMs = silenceDurationMs;
+ }
+
+ public Float getThreshold() {
+ return threshold;
+ }
+
+ public void setThreshold(Float threshold) {
+ this.threshold = threshold;
+ }
+
+ public Boolean getCreateResponse() {
+ return createResponse;
+ }
+
+ public void setCreateResponse(Boolean createResponse) {
+ this.createResponse = createResponse;
+ }
+
+ public Boolean getInterruptResponse() {
+ return interruptResponse;
+ }
+
+ public void setInterruptResponse(Boolean interruptResponse) {
+ this.interruptResponse = interruptResponse;
+ }
+
+ public String getEagerness() {
+ return eagerness;
+ }
+
+ public void setEagerness(String eagerness) {
+ this.eagerness = eagerness;
+ }
+
+
public static TurnDetectionPayload.Builder builder() {
return new TurnDetectionPayload.Builder();
}
private TurnDetectionPayload(Builder builder) {
setInterruptMode(builder.interruptMode);
+ setType(builder.type);
+ setInterruptDurationMs(builder.interruptDurationMs);
+ setPrefixPaddingMs(builder.prefixPaddingMs);
+ setSilenceDurationMs(builder.silenceDurationMs);
+ setThreshold(builder.threshold);
+ setCreateResponse(builder.createResponse);
+ setInterruptResponse(builder.interruptResponse);
+ setEagerness(builder.eagerness);
}
public static final class Builder {
private String interruptMode;
+ private String type;
+ private Integer interruptDurationMs;
+ private Integer prefixPaddingMs;
+ private Integer silenceDurationMs;
+ private Float threshold;
+ private Boolean createResponse;
+ private Boolean interruptResponse;
+ private String eagerness;
+
private Builder() {
}
@@ -2465,9 +3815,51 @@ public Builder interruptMode(String val) {
return this;
}
+ public Builder type(String val) {
+ type = val;
+ return this;
+ }
+
+ public Builder interruptDurationMs(Integer val) {
+ interruptDurationMs = val;
+ return this;
+ }
+
+ public Builder prefixPaddingMs(Integer val) {
+ prefixPaddingMs = val;
+ return this;
+ }
+
+ public Builder silenceDurationMs(Integer val) {
+ silenceDurationMs = val;
+ return this;
+ }
+
+ public Builder threshold(Float val) {
+ threshold = val;
+ return this;
+ }
+
+ public Builder interruptResponse(Boolean val) {
+ interruptResponse = val;
+ return this;
+ }
+
+ public Builder eagerness(String val) {
+ eagerness = val;
+ return this;
+ }
+
+ public Builder createResponse(Boolean val) {
+ createResponse = val;
+ return this;
+ }
+
+
public TurnDetectionPayload build() {
return new TurnDetectionPayload(this);
}
+
}
}
@@ -2565,9 +3957,55 @@ public Parameters build() {
*/
public static class FixedParams {
+ /**
+ * Silence config
+ *
+ * @since v0.6.0
+ */
@JsonProperty("silence_config")
private SilenceConfig silenceConfig;
+ /**
+ * Agent data transmission channel(Optional):
+ *
+ * - "rtm": Use RTM transmission. This configuration takes effect only when advanced_features.enable_rtm is true.
+ *
+ * - "datastream": Use RTC data stream transport.(Default)
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("data_channel")
+ private String dataChannel;
+
+
+ /**
+ * Whether to receive agent performance data(Optional):
+ *
+ * - true: Receive agent performance data.
+ *
+ * - false: Do not receive agent performance data.(Default)
+ *
+ * This setting only takes effect when advanced_features.enable_rtm is true.
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("enable_metrics")
+ private Boolean enableMetrics;
+
+ /**
+ * Whether to receive agent error events(Optional):
+ *
+ * - true: Receive agent error events.
+ *
+ * - false: Do not receive agent error events.(Default)
+ *
+ * This setting only takes effect when advanced_features.enable_rtm is true
+ *
+ * @since v0.7.0
+ */
+ @JsonProperty("enable_error_message")
+ private Boolean enableErrorMessage;
+
public SilenceConfig getSilenceConfig() {
return silenceConfig;
}
@@ -2576,16 +4014,36 @@ public void setSilenceConfig(SilenceConfig silenceConfig) {
this.silenceConfig = silenceConfig;
}
+ public Boolean getEnableMetrics() {
+ return enableMetrics;
+ }
+
+ public Boolean getEnableErrorMessage() {
+ return enableErrorMessage;
+ }
+
+ public void setEnableMetrics(Boolean enableMetrics) {
+ this.enableMetrics = enableMetrics;
+ }
+
+ public void setEnableErrorMessage(Boolean enableErrorMessage) {
+ this.enableErrorMessage = enableErrorMessage;
+ }
+
public static Builder builder() {
return new Builder();
}
private FixedParams(Builder builder) {
setSilenceConfig(builder.silenceConfig);
+ setEnableMetrics(builder.enableMetrics);
+ setEnableErrorMessage(builder.enableErrorMessage);
}
public static final class Builder {
private SilenceConfig silenceConfig;
+ private Boolean enableMetrics;
+ private Boolean enableErrorMessage;
private Builder() {
}
@@ -2595,6 +4053,16 @@ public Builder silenceConfig(SilenceConfig val) {
return this;
}
+ public Builder enableMetrics(Boolean val) {
+ enableMetrics = val;
+ return this;
+ }
+
+ public Builder enableErrorMessage(Boolean val) {
+ enableErrorMessage = val;
+ return this;
+ }
+
public FixedParams build() {
return new FixedParams(this);
}
diff --git a/examples/convoai/README.md b/examples/convoai/README.md
index 4ac26f7..225657d 100644
--- a/examples/convoai/README.md
+++ b/examples/convoai/README.md
@@ -37,6 +37,25 @@ export CONVOAI_TTS_ELEVENLABS_MODEL_ID=