Skip to content

Commit a3e35d5

Browse files
PaulAsjesPaulAsjes
committed
client tools
1 parent 9feeb70 commit a3e35d5

File tree

7 files changed

+144
-46
lines changed

7 files changed

+144
-46
lines changed

‎elevenlabs-sdk/src/main/java/com/elevenlabs/ConversationClientImpl.kt‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,15 @@ internal object ConversationClientImpl {
5353
val audioManager = LiveKitAudioManager(context, room)
5454
Log.d("ConversationClient", "LiveKitAudioManager initialized")
5555

56-
// Create client tools registry with common tools
57-
val toolRegistry = ClientToolRegistryBuilder()
58-
.addTool("get_current_time", CommonClientTools.getCurrentTime)
59-
.addTool("get_device_info", CommonClientTools.getDeviceInfo)
60-
.addTool("log_message", CommonClientTools.logMessage)
61-
.build()
56+
// Create client tools registry from configuration
57+
val toolRegistry = ClientToolRegistry()
58+
finalConfig.clientTools.forEach { (name, tool) ->
59+
try {
60+
toolRegistry.registerTool(name, tool)
61+
} catch (e: Exception) {
62+
Log.d("ConversationClient", "Failed to register client tool '$name': ${e.message}")
63+
}
64+
}
6265

6366
// Configure audio session for conversation
6467
if (!finalConfig.textOnly) {

‎elevenlabs-sdk/src/main/java/com/elevenlabs/ConversationConfig.kt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.elevenlabs
22

3+
import com.elevenlabs.models.ConversationEvent.ClientToolCall
4+
35
/**
46
* Configuration class for conversation sessions
57
*
@@ -28,9 +30,13 @@ data class ConversationConfig(
2830
val overrides: Overrides? = null,
2931
val customLlmExtraBody: Map<String, Any>? = null,
3032
val dynamicVariables: Map<String, Any>? = null,
33+
val clientTools: Map<String, ClientTool> = emptyMap(),
3134
val onConnect: ((conversationId: String) -> Unit)? = null,
3235
val onMessage: ((source: String, message: String) -> Unit)? = null,
33-
val onModeChange: ((mode: String) -> Unit)? = null
36+
val onModeChange: ((mode: String) -> Unit)? = null,
37+
val onStatusChange: ((status: String) -> Unit)? = null,
38+
val onCanSendFeedbackChange: ((canSend: Boolean) -> Unit)? = null,
39+
val onUnhandledClientToolCall: ((ClientToolCall) -> Unit)? = null
3440

3541
) {
3642
init {

‎elevenlabs-sdk/src/main/java/com/elevenlabs/ConversationEventHandler.kt‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ import kotlinx.coroutines.flow.StateFlow
1717
class ConversationEventHandler(
1818
private val audioManager: AudioManager,
1919
private val toolRegistry: ClientToolRegistry,
20-
private val messageCallback: (OutgoingEvent) -> Unit
20+
private val messageCallback: (OutgoingEvent) -> Unit,
21+
private val onCanSendFeedbackChange: ((Boolean) -> Unit)? = null,
22+
private val onUnhandledClientToolCall: ((ConversationEvent.ClientToolCall) -> Unit)? = null
2123
) {
2224

2325
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
@@ -73,6 +75,7 @@ class ConversationEventHandler(
7375

7476
// Store the last agent event ID for feedback
7577
_lastAgentEventId.value = event.eventId
78+
onCanSendFeedbackChange?.invoke(true)
7679

7780
// If this is a voice conversation, ensure audio playback is active
7881
if (!audioManager.isPlaying()) {
@@ -110,6 +113,7 @@ class ConversationEventHandler(
110113
private fun handleInterruption(event: ConversationEvent.Interruption) {
111114
// User interrupted agent speech - switch to listening mode
112115
_conversationMode.value = ConversationMode.LISTENING
116+
onCanSendFeedbackChange?.invoke(false)
113117

114118
Log.d("ConversationEventHandler", "Conversation interrupted: ${event.eventId}")
115119
}
@@ -119,9 +123,18 @@ class ConversationEventHandler(
119123
*/
120124
private suspend fun handleClientToolCall(event: ConversationEvent.ClientToolCall) {
121125
scope.launch {
126+
val toolExists = toolRegistry.isToolRegistered(event.toolName)
127+
if (!toolExists) {
128+
// Notify app layer about unhandled tool call
129+
try { onUnhandledClientToolCall?.invoke(event) } catch (_: Throwable) {}
130+
}
131+
122132
val result = try {
123-
// Execute the requested tool
124-
toolRegistry.executeTool(event.toolName, event.parameters)
133+
if (!toolExists) {
134+
ClientToolResult.failure("Tool '${event.toolName}' not registered on client")
135+
} else {
136+
toolRegistry.executeTool(event.toolName, event.parameters)
137+
}
125138
} catch (e: Exception) {
126139
ClientToolResult.failure("Tool execution failed: ${e.message}")
127140
}
@@ -140,8 +153,7 @@ class ConversationEventHandler(
140153

141154
messageCallback(toolResultEvent)
142155
}
143-
144-
Log.d("ConversationEventHandler", "Tool executed: ${event.toolName} -> ${if (result.success) "SUCCESS" else "FAILED"}")
156+
Log.d("ConversationEventHandler", "Tool executed: ${event.toolName} -> ${if (result.success) "SUCCESS" else "FAILED"}")
145157
}
146158
}
147159

@@ -202,6 +214,7 @@ class ConversationEventHandler(
202214
}
203215
}
204216
_conversationMode.value = ConversationMode.LISTENING
217+
onCanSendFeedbackChange?.invoke(false)
205218
}
206219
ConversationStatus.ERROR -> {
207220
// Connection error - handle gracefully

‎elevenlabs-sdk/src/main/java/com/elevenlabs/ConversationSessionImpl.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ internal class ConversationSessionImpl(
4040
messageCallback = { event ->
4141
// Send outgoing events through the connection
4242
connection.sendMessage(event)
43+
},
44+
onCanSendFeedbackChange = { canSend ->
45+
try { config.onCanSendFeedbackChange?.invoke(canSend) } catch (_: Throwable) {}
46+
},
47+
onUnhandledClientToolCall = { call ->
48+
try { config.onUnhandledClientToolCall?.invoke(call) } catch (_: Throwable) {}
4349
}
4450
)
4551

‎elevenlabs-sdk/src/main/java/com/elevenlabs/network/ConversationEventParser.kt‎

Lines changed: 66 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ object ConversationEventParser {
4141
"connection_state_change" -> parseConnectionStateChange(jsonObject)
4242
"error" -> parseError(jsonObject)
4343
"ping" -> parsePing(jsonObject)
44+
"agent_tool_response" -> {
45+
// Ignore quietly; this is the server acknowledging a tool result
46+
logAgentToolResponse(jsonObject)
47+
null
48+
}
4449
else -> {
4550
handleParsingError(json, IllegalArgumentException("Unknown event type: $eventType"))
4651
null
@@ -84,22 +89,62 @@ object ConversationEventParser {
8489
* Parse agent response event
8590
*/
8691
private fun parseAgentResponse(jsonObject: JsonObject): ConversationEvent.AgentResponse {
92+
val obj = jsonObject.getAsJsonObject("agent_response_event")
93+
?: jsonObject.getAsJsonObject("agent_response")
94+
?: jsonObject
95+
96+
val content = when {
97+
obj.get("agent_response") != null && !obj.get("agent_response").isJsonNull -> obj.get("agent_response").asString
98+
obj.get("content") != null && !obj.get("content").isJsonNull -> obj.get("content").asString
99+
obj.get("text") != null && !obj.get("text").isJsonNull -> obj.get("text").asString
100+
obj.get("message") != null && !obj.get("message").isJsonNull -> obj.get("message").asString
101+
else -> ""
102+
}
103+
104+
val eventId = obj.get("event_id")?.asString?.takeIf { it.isNotBlank() }
105+
?: "evt_${System.currentTimeMillis()}"
106+
107+
val timestamp = obj.get("timestamp")?.asLong ?: System.currentTimeMillis()
108+
87109
return ConversationEvent.AgentResponse(
88-
content = jsonObject.get("content")?.asString ?: "",
89-
eventId = jsonObject.get("event_id")?.asString ?: "",
90-
timestamp = jsonObject.get("timestamp")?.asLong ?: System.currentTimeMillis()
110+
content = content,
111+
eventId = eventId,
112+
timestamp = timestamp
91113
)
92114
}
93115

94116
/**
95117
* Parse user transcript event
96118
*/
97119
private fun parseUserTranscript(jsonObject: JsonObject): ConversationEvent.UserTranscript {
120+
val obj = jsonObject.getAsJsonObject("user_transcription_event")
121+
?: jsonObject.getAsJsonObject("user_transcript")
122+
?: jsonObject
123+
124+
val content = when {
125+
obj.get("user_transcript") != null && !obj.get("user_transcript").isJsonNull -> obj.get("user_transcript").asString
126+
obj.get("content") != null && !obj.get("content").isJsonNull -> obj.get("content").asString
127+
obj.get("text") != null && !obj.get("text").isJsonNull -> obj.get("text").asString
128+
obj.get("transcript") != null && !obj.get("transcript").isJsonNull -> obj.get("transcript").asString
129+
else -> ""
130+
}
131+
132+
val eventId = obj.get("event_id")?.asString?.takeIf { it.isNotBlank() }
133+
?: "evt_${System.currentTimeMillis()}"
134+
135+
val timestamp = obj.get("timestamp")?.asLong ?: System.currentTimeMillis()
136+
137+
val isFinal = when {
138+
obj.get("is_final") != null && !obj.get("is_final").isJsonNull -> obj.get("is_final").asBoolean
139+
obj.get("final") != null && !obj.get("final").isJsonNull -> obj.get("final").asBoolean
140+
else -> true
141+
}
142+
98143
return ConversationEvent.UserTranscript(
99-
content = jsonObject.get("content")?.asString ?: "",
100-
eventId = jsonObject.get("event_id")?.asString ?: "",
101-
timestamp = jsonObject.get("timestamp")?.asLong ?: System.currentTimeMillis(),
102-
isFinal = jsonObject.get("is_final")?.asBoolean ?: true
144+
content = content,
145+
eventId = eventId,
146+
timestamp = timestamp,
147+
isFinal = isFinal
103148
)
104149
}
105150

@@ -117,7 +162,10 @@ object ConversationEventParser {
117162
* Parse client tool call event
118163
*/
119164
private fun parseClientToolCall(jsonObject: JsonObject): ConversationEvent.ClientToolCall {
120-
val parametersJson = jsonObject.get("parameters")?.asJsonObject
165+
// Payloads can be either flat or nested under "client_tool_call"
166+
val obj = jsonObject.getAsJsonObject("client_tool_call") ?: jsonObject
167+
168+
val parametersJson = obj.get("parameters")?.asJsonObject
121169
val parameters = mutableMapOf<String, Any>()
122170

123171
parametersJson?.entrySet()?.forEach { entry ->
@@ -138,14 +186,20 @@ object ConversationEventParser {
138186
}
139187

140188
return ConversationEvent.ClientToolCall(
141-
toolName = jsonObject.get("tool_name")?.asString ?: "",
189+
toolName = obj.get("tool_name")?.asString ?: "",
142190
parameters = parameters,
143-
toolCallId = jsonObject.get("tool_call_id")?.asString ?: "",
144-
expectsResponse = jsonObject.get("expects_response")?.asBoolean ?: true,
145-
timestamp = jsonObject.get("timestamp")?.asLong ?: System.currentTimeMillis()
191+
toolCallId = obj.get("tool_call_id")?.asString ?: "",
192+
expectsResponse = obj.get("expects_response")?.asBoolean ?: true,
193+
timestamp = obj.get("timestamp")?.asLong ?: System.currentTimeMillis()
146194
)
147195
}
148196

197+
private fun logAgentToolResponse(jsonObject: JsonObject) {
198+
try {
199+
Log.d("ConversationEventParser", "Agent tool response: ${jsonObject}")
200+
} catch (_: Exception) { }
201+
}
202+
149203
/**
150204
* Parse mode change event
151205
*/

‎elevenlabs-sdk/src/main/java/com/elevenlabs/network/WebRTCConnection.kt‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ class WebRTCConnection(
7373
// Start message processing
7474
startMessageProcessing()
7575

76+
// This is updated here rather than in the Connected event handler to ensure that the connection state is updated before the overrides are sent
7677
updateConnectionState(ConnectionState.CONNECTED)
77-
Log.d("WebRTCConnection", "Connected. roomSid=${room.sid}, name=${room.name}")
7878

7979
// Send initiation overrides payload after connect
8080
try {
@@ -162,7 +162,7 @@ class WebRTCConnection(
162162
room.events.collect { event ->
163163
when (event) {
164164
is RoomEvent.Connected -> {
165-
Log.d("WebRTCConnection", "LiveKit room connected (sid=${room.sid})")
165+
Log.d("WebRTCConnection", "Connected. roomSid=${room.sid}, name=${room.name}")
166166
// invoke user callback if provided with extracted conversation id
167167
try {
168168
val roomName = room.name ?: ""
@@ -211,7 +211,7 @@ class WebRTCConnection(
211211
}
212212

213213
else -> {
214-
// Handle other events as needed
214+
// Log.d("WebRTCConnection", "Unhandled event: ${event.javaClass.simpleName}")
215215
}
216216
}
217217
}
@@ -295,25 +295,19 @@ class WebRTCConnection(
295295
if (_connectionState.value != newState) {
296296
_connectionState.value = newState
297297
connectionStateListener?.invoke(newState)
298+
// Invoke user status change callback if provided
299+
val statusString = when (newState) {
300+
ConnectionState.CONNECTED -> "connected"
301+
ConnectionState.CONNECTING -> "connecting"
302+
ConnectionState.DISCONNECTED, ConnectionState.ERROR, ConnectionState.RECONNECTING -> "disconnected"
303+
else -> "disconnected"
304+
}
305+
try {
306+
latestConfig?.onStatusChange?.invoke(statusString)
307+
} catch (_: Throwable) { }
298308
}
299309
}
300310

301-
/**
302-
* Get local audio track for microphone input
303-
* TODO: Implement proper audio track access when LiveKit API is clarified
304-
*/
305-
fun getLocalAudioTrack(): LocalAudioTrack? {
306-
return null // Placeholder implementation
307-
}
308-
309-
/**
310-
* Get remote audio tracks for agent speech
311-
* TODO: Implement proper remote audio track access when LiveKit API is clarified
312-
*/
313-
fun getRemoteAudioTracks(): List<RemoteAudioTrack> {
314-
return emptyList() // Placeholder implementation
315-
}
316-
317311
/**
318312
* Clean up resources when the connection is no longer needed
319313
*/

‎example-app/src/main/java/com/elevenlabs/example/viewmodels/ConversationViewModel.kt‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,37 @@ class ConversationViewModel(application: Application) : AndroidViewModel(applica
6363
overrides = null,
6464
customLlmExtraBody = null,
6565
dynamicVariables = null,
66+
clientTools = mapOf(
67+
"logMessage" to object : com.elevenlabs.ClientTool {
68+
override suspend fun execute(parameters: Map<String, Any>): com.elevenlabs.ClientToolResult {
69+
val message = parameters["message"] as? String
70+
?: return com.elevenlabs.ClientToolResult.failure("Missing 'message' parameter")
71+
val level = parameters["level"] as? String ?: "INFO"
72+
73+
Log.d("ExampleApp", "[$level] Client Tool Log: $message")
74+
return com.elevenlabs.ClientToolResult.success("Message logged successfully")
75+
}
76+
}
77+
),
6678
onConnect = { conversationId ->
6779
Log.d("ConversationViewModel", "Connected id=$conversationId")
6880
},
69-
onMessage = { source, message ->
70-
Log.d("ConversationViewModel", "onMessage [$source]: $message")
71-
},
81+
// onMessage = { source, message ->
82+
// Log.d("ConversationViewModel", "onMessage [$source]: $message")
83+
// },
7284
onModeChange = { mode ->
7385
_mode.postValue(mode)
86+
},
87+
onStatusChange = { status ->
88+
Log.d("ConversationViewModel", "onStatusChange: $status")
89+
},
90+
onCanSendFeedbackChange = { canSendFeedback ->
91+
Log.d("ConversationViewModel", "onCanSendFeedbackChange: $canSendFeedback")
92+
},
93+
onUnhandledClientToolCall = { toolCall ->
94+
Log.d("ConversationViewModel", "onUnhandledClientToolCall: $toolCall")
7495
}
96+
7597
)
7698

7799
val session = ConversationClient.startSession(config, activityContext)

0 commit comments

Comments
 (0)