Skip to content

Commit 889babc

Browse files
VinayGuthalVinayGuthal
committed
commited
1 parent c32ce9a commit 889babc

File tree

9 files changed

+302
-125
lines changed

9 files changed

+302
-125
lines changed

‎firebase-vertexai/gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
version=16.3.0
15+
version=17.1.0
1616
latestReleasedVersion=16.2.0

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/LiveGenerativeModel.kt‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import io.ktor.websocket.readBytes
3535
import kotlinx.serialization.Serializable
3636
import kotlinx.serialization.encodeToString
3737
import kotlinx.serialization.json.Json
38+
import kotlin.system.exitProcess
3839

3940
/**
4041
* Represents a multimodal model (like Gemini), capable of generating content based on various input
@@ -74,10 +75,16 @@ internal constructor(
7475
),
7576
)
7677

78+
public fun getModelName(): String {
79+
return this.modelName
80+
}
81+
7782
@Serializable
7883
internal data class BidiGenerateContentSetup(
7984
val model: String,
80-
val generation_config: LiveGenerationConfig.Internal?
85+
val generationConfig: LiveGenerationConfig.Internal?,
86+
val tools: List<Tool.Internal>?,
87+
val systemInstruction: Content.Internal?
8188
)
8289

8390
@Serializable
@@ -87,7 +94,8 @@ internal constructor(
8794
val client = HttpClient(CIO) { install(WebSockets) }
8895

8996
val roundedUrl = this.controller.getBidiEndpoint()
90-
val setup = BidiGenerateContentSetup(this.modelName, this.config?.toInternal())
97+
val setup = BidiGenerateContentSetup(this.modelName, this.config?.toInternal(),
98+
this.tools?.map{it.toInternal()}, this.systemInstruction?.toInternal())
9199
val data: String = Json.encodeToString(BidiGenerateContentClientMessage(setup))
92100
val webSession = client.webSocketSession(roundedUrl)
93101
webSession.send(Frame.Text(data))

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/AudioHelper.kt‎

Lines changed: 78 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -9,86 +9,93 @@ import android.media.audiofx.AcousticEchoCanceler
99
import kotlinx.coroutines.flow.Flow
1010
import kotlinx.coroutines.flow.flow
1111

12-
13-
1412
internal class AudioHelper {
1513

16-
private lateinit var audioRecord: AudioRecord
17-
private lateinit var audioTrack: AudioTrack
18-
private var stopRecording: Boolean = false
19-
private val RECORDER_SAMPLE_RATE = 16000 // Adjust based on server settings
20-
private val RECORDER_CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
21-
private val RECORDER_AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
14+
private lateinit var audioRecord: AudioRecord
15+
private lateinit var audioTrack: AudioTrack
16+
private var stopRecording: Boolean = false
17+
private val RECORDER_SAMPLE_RATE = 16000 // Adjust based on server settings
18+
private val RECORDER_CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
19+
private val RECORDER_AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
2220

23-
internal fun release() {
24-
stopRecording = true
25-
if(::audioRecord.isInitialized) {
26-
audioRecord.stop()
27-
audioRecord.release()
28-
}
29-
if(::audioTrack.isInitialized) {
30-
audioTrack.stop()
31-
audioTrack.release()
32-
}
21+
internal fun release() {
22+
stopRecording = true
23+
if (::audioRecord.isInitialized) {
24+
audioRecord.stop()
25+
audioRecord.release()
3326
}
34-
35-
internal fun setupAudioTrack() {
36-
val sampleRate = 24000 // Adjust based on server settings
37-
val channelConfig = AudioFormat.CHANNEL_OUT_MONO
38-
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
39-
val minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig, audioFormat)
40-
audioTrack = AudioTrack(
41-
AudioManager.STREAM_MUSIC,
42-
sampleRate,
43-
channelConfig,
44-
audioFormat,
45-
minBufferSize,
46-
AudioTrack.MODE_STREAM
47-
)
48-
audioTrack.play()
27+
if (::audioTrack.isInitialized) {
28+
audioTrack.stop()
29+
audioTrack.release()
4930
}
31+
}
5032

51-
internal fun playAudio(data: ByteArray) {
52-
if(!stopRecording) {
53-
audioTrack.write(data, 0, data.size)
54-
}
33+
internal fun setupAudioTrack() {
34+
val sampleRate = 24000 // Adjust based on server settings
35+
val channelConfig = AudioFormat.CHANNEL_OUT_MONO
36+
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
37+
val minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig, audioFormat)
38+
audioTrack =
39+
AudioTrack(
40+
AudioManager.STREAM_MUSIC,
41+
sampleRate,
42+
channelConfig,
43+
audioFormat,
44+
minBufferSize,
45+
AudioTrack.MODE_STREAM
46+
)
47+
audioTrack.play()
48+
}
49+
50+
internal fun playAudio(data: ByteArray) {
51+
if (!stopRecording) {
52+
audioTrack.write(data, 0, data.size)
5553
}
54+
}
5655

57-
suspend fun startRecording(): Flow<ByteArray> {
58-
val bufferSize = AudioRecord.getMinBufferSize(RECORDER_SAMPLE_RATE, RECORDER_CHANNEL_CONFIG, RECORDER_AUDIO_FORMAT)
59-
if (bufferSize == AudioRecord.ERROR || bufferSize == AudioRecord.ERROR_BAD_VALUE || bufferSize <= 0) {
60-
println("Invalid buffer size: $bufferSize")
61-
}
62-
audioRecord = AudioRecord(
63-
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
64-
RECORDER_SAMPLE_RATE,
65-
RECORDER_CHANNEL_CONFIG,
66-
RECORDER_AUDIO_FORMAT,
67-
bufferSize
68-
)
69-
if (audioRecord.state != AudioRecord.STATE_INITIALIZED) {
70-
println("AudioRecord initialization failed.")
71-
}
72-
if (AcousticEchoCanceler.isAvailable()) {
73-
val echoCanceler = AcousticEchoCanceler.create(audioRecord.audioSessionId)
74-
echoCanceler?.enabled = true
75-
}
56+
suspend fun startRecording(): Flow<ByteArray> {
57+
val bufferSize =
58+
AudioRecord.getMinBufferSize(
59+
RECORDER_SAMPLE_RATE,
60+
RECORDER_CHANNEL_CONFIG,
61+
RECORDER_AUDIO_FORMAT
62+
)
63+
if (
64+
bufferSize == AudioRecord.ERROR ||
65+
bufferSize == AudioRecord.ERROR_BAD_VALUE ||
66+
bufferSize <= 0
67+
) {
68+
println("Invalid buffer size: $bufferSize")
69+
}
70+
audioRecord =
71+
AudioRecord(
72+
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
73+
RECORDER_SAMPLE_RATE,
74+
RECORDER_CHANNEL_CONFIG,
75+
RECORDER_AUDIO_FORMAT,
76+
bufferSize
77+
)
78+
if (audioRecord.state != AudioRecord.STATE_INITIALIZED) {
79+
println("AudioRecord initialization failed.")
80+
}
81+
if (AcousticEchoCanceler.isAvailable()) {
82+
val echoCanceler = AcousticEchoCanceler.create(audioRecord.audioSessionId)
83+
echoCanceler?.enabled = true
84+
}
7685

77-
audioRecord.startRecording()
86+
audioRecord.startRecording()
7887

79-
return flow {
80-
while(true) {
81-
if(stopRecording) {
82-
break
83-
}
84-
val buffer = ByteArray(bufferSize/2)
85-
val bytesRead = audioRecord.read(
86-
buffer, 0, buffer.size
87-
)
88-
if(bytesRead>0) {
89-
emit(buffer.copyOf(bytesRead))
90-
}
91-
}
88+
return flow {
89+
while (true) {
90+
if (stopRecording) {
91+
break
92+
}
93+
val buffer = ByteArray(bufferSize / 2)
94+
val bytesRead = audioRecord.read(buffer, 0, buffer.size)
95+
if (bytesRead > 0) {
96+
emit(buffer.copyOf(bytesRead))
9297
}
98+
}
9399
}
94-
}
100+
}
101+
}

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/LiveContentResponse.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.google.firebase.vertexai.type
2+
3+
4+
5+
public class LiveContentResponse internal constructor(public val data: Content?, public val interrupted: Boolean?, public val functionCalls: List<FunctionCallPart>?) {
6+
/**
7+
* Convenience field representing all the text parts in the response as a single string, if they
8+
* exists.
9+
*/
10+
11+
public val text: String? = data?.parts?.filterIsInstance<TextPart>()?.joinToString(" ") { it.text }
12+
13+
}

0 commit comments

Comments
 (0)