diff --git a/ai-catalog/README.md b/ai-catalog/README.md index 416f2373..86369866 100644 --- a/ai-catalog/README.md +++ b/ai-catalog/README.md @@ -21,6 +21,7 @@ Browse the samples inside the `/samples` folder: - **imagen**: an image generation sample using Imagen - **magic-selfie**: an sample using ML Kit subject segmentation and Imagen for image generation - **gemini-video-summarization**: a video summarization sample using Gemini 2.0 Flash +- **gemini-live-todo**: a todo list app using Gemini Live - More to come... > **Requires Firebase setup** the samples relying on Google Cloud models (Gemini Pro, Gemini Flash, etc...) diff --git a/ai-catalog/app/build.gradle.kts b/ai-catalog/app/build.gradle.kts index 0170570f..7e46c13d 100644 --- a/ai-catalog/app/build.gradle.kts +++ b/ai-catalog/app/build.gradle.kts @@ -88,6 +88,7 @@ dependencies { implementation(project(":samples:imagen")) implementation(project(":samples:magic-selfie")) implementation(project(":samples:gemini-video-summarization")) + implementation(project(":samples:gemini-live-todo")) testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) diff --git a/ai-catalog/app/src/main/AndroidManifest.xml b/ai-catalog/app/src/main/AndroidManifest.xml index 8a8f61f1..c0b29615 100644 --- a/ai-catalog/app/src/main/AndroidManifest.xml +++ b/ai-catalog/app/src/main/AndroidManifest.xml @@ -2,6 +2,8 @@ + + ( +@androidx.annotation.RequiresPermission(android.Manifest.permission.RECORD_AUDIO) +val sampleCatalog = listOf( SampleCatalogItem( title = R.string.gemini_multimodal_sample_title, description = R.string.gemini_multimodal_sample_description, @@ -90,6 +92,14 @@ val sampleCatalog = listOf( tags = listOf(SampleTags.GEMINI_2_0_FLASH, SampleTags.FIREBASE, SampleTags.MEDIA3), needsFirebase = true, ), + SampleCatalogItem( + title = R.string.gemini_live_todo_title, + description = R.string.gemini_live_todo_description, + route = "GeminiLiveTodoScreen", + sampleEntryScreen = { TodoScreen() }, + tags = listOf(SampleTags.GEMINI_2_0_FLASH, SampleTags.FIREBASE), + needsFirebase = true, + ), // To create a new sample entry, add a new SampleCatalogItem here. ) diff --git a/ai-catalog/app/src/main/res/values/strings.xml b/ai-catalog/app/src/main/res/values/strings.xml index d867866b..a01b888a 100644 --- a/ai-catalog/app/src/main/res/values/strings.xml +++ b/ai-catalog/app/src/main/res/values/strings.xml @@ -18,6 +18,8 @@ Change the background of your selfies with Imagen and the ML Kit Segmentation API Video Summarization with Gemini and Firebase "Generate a summary of a video (from a cloud URL or Youtube) with Gemini API powered by Firebase" + Gemini Live Todo + "Simple Todo app using the Gemini Live API to interact with the items in the list" Firebase Required This feature requires Firebase to be initialized. Close diff --git a/ai-catalog/gradle/libs.versions.toml b/ai-catalog/gradle/libs.versions.toml index 411ab97d..f822b41b 100644 --- a/ai-catalog/gradle/libs.versions.toml +++ b/ai-catalog/gradle/libs.versions.toml @@ -29,6 +29,10 @@ uiToolingPreviewAndroid = "1.8.1" spotless = "7.0.4" uiToolingPreview = "1.8.3" uiTooling = "1.8.3" +material = "1.10.0" +firebaseAi = "16.2.0" +lifecycleViewmodelAndroid = "2.8.7" +material3 = "1.3.2" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -54,6 +58,7 @@ androidx-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-toolin androidx-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-manifest" } androidx-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4" } androidx-material3 = { group = "androidx.compose.material3", name = "material3" } +androidx-material3-android = { group = "androidx.compose.material3", name = "material3-android", version.ref = "material3Android" } androidx-material-icons-extended = { module = "androidx.compose.material:material-icons-extended" } androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" } androidx-navigation-runtime-ktx = { group = "androidx.navigation", name = "navigation-runtime-ktx", version.ref = "navigationRuntimeKtx" } @@ -64,12 +69,14 @@ hilt-android = { group = "com.google.dagger", name = "hilt-android", version.ref hilt-compiler = { group = "com.google.dagger", name = "hilt-android-compiler", version.ref = "hilt"} hilt-navigation-compose = { group = "androidx.hilt", name = "hilt-navigation-compose", version.ref = "hiltNavigationCompose" } androidx-runtime-livedata = { group = "androidx.compose.runtime", name = "runtime-livedata", version.ref = "runtimeLivedata" } -androidx-material3-android = { group = "androidx.compose.material3", name = "material3-android", version.ref = "material3Android" } androidx-media3-exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" } androidx-media3-ui = { module = "androidx.media3:media3-ui", version.ref = "media3" } androidx-ui-tooling-preview-android = { group = "androidx.compose.ui", name = "ui-tooling-preview-android", version.ref = "uiToolingPreviewAndroid" } ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "uiToolingPreview" } ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "uiTooling" } +google-firebase-ai = { group = "com.google.firebase", name = "firebase-ai", version.ref = "firebaseAi" } +androidx-lifecycle-viewmodel-android = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-android", version.ref = "lifecycleViewmodelAndroid" } +material3 = { group = "androidx.compose.material3", name = "material3", version.ref = "material3" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/ai-catalog/samples/gemini-live-todo/.gitignore b/ai-catalog/samples/gemini-live-todo/.gitignore new file mode 100644 index 00000000..42afabfd --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/ai-catalog/samples/gemini-live-todo/build.gradle.kts b/ai-catalog/samples/gemini-live-todo/build.gradle.kts new file mode 100644 index 00000000..8f8b6542 --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/build.gradle.kts @@ -0,0 +1,71 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.jetbrains.kotlin.android) + alias(libs.plugins.ksp) + alias(libs.plugins.compose.compiler) +} + +android { + namespace = "com.android.ai.samples.geminilivetodo" + compileSdk = 35 + + defaultConfig { + minSdk = 24 + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro", + ) + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + kotlinOptions { + jvmTarget = "17" + } +} + +dependencies { + + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.appcompat) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.material.icons.extended) + implementation(platform(libs.firebase.bom)) + implementation(libs.google.firebase.ai) + implementation(libs.androidx.lifecycle.viewmodel.android) + implementation(libs.material3) + implementation(libs.hilt.android) + ksp(libs.hilt.compiler) + implementation(libs.hilt.navigation.compose) + implementation(libs.androidx.activity.compose) + implementation(libs.androidx.material3.android) + implementation(libs.kotlinx.serialization.json) + testImplementation(libs.junit) + androidTestImplementation(libs.androidx.junit) + androidTestImplementation(libs.androidx.espresso.core) +} diff --git a/ai-catalog/samples/gemini-live-todo/consumer-rules.pro b/ai-catalog/samples/gemini-live-todo/consumer-rules.pro new file mode 100644 index 00000000..e69de29b diff --git a/ai-catalog/samples/gemini-live-todo/proguard-rules.pro b/ai-catalog/samples/gemini-live-todo/proguard-rules.pro new file mode 100644 index 00000000..481bb434 --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt new file mode 100644 index 00000000..e3e1e8dd --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ai.samples.geminilivetodo.data + +import java.util.UUID.randomUUID + +data class Todo( + val id: Long = randomUUID().mostSignificantBits, + val task: String, + val isCompleted: Boolean = false, +) diff --git a/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt new file mode 100644 index 00000000..f28b2c4c --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ai.samples.geminilivetodo.data + +import javax.inject.Inject +import javax.inject.Singleton +import kotlin.collections.filterNot +import kotlin.collections.map +import kotlin.collections.plus +import kotlin.text.isNotBlank +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update + +@Singleton +class TodoRepository @Inject constructor() { + + private val _todos = MutableStateFlow>( + listOf( + Todo(1234, "buy bread", false), + Todo(1235, "do the dishes", false), + Todo(1236, "buy eggs", false), + Todo(1237, "read a book", false), + ), + ) + val todos: Flow> = _todos.asStateFlow() + + fun getTodoList(): List = _todos.value + + fun addTodo(taskDescription: String) { + if (taskDescription.isNotBlank()) { + val newTodo = Todo(task = taskDescription) + _todos.update { currentList -> + currentList + newTodo + } + } + } + + fun removeTodo(todoId: Long) { + _todos.update { currentList -> + currentList.filterNot { it.id == todoId } + } + } + + fun toggleTodoStatus(todoId: Long) { + _todos.update { currentList -> + currentList.map { todo -> + if (todo.id == todoId) { + todo.copy(isCompleted = !todo.isCompleted) + } else { + todo + } + } + } + } +} diff --git a/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt new file mode 100644 index 00000000..3de62179 --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt @@ -0,0 +1,285 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ai.samples.geminilivetodo.ui + +import android.app.Activity +import androidx.activity.compose.LocalActivity +import androidx.compose.animation.Animatable +import androidx.compose.animation.animateColor +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.itemsIndexed +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Delete +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.MicNone +import androidx.compose.material.icons.filled.MicOff +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.Checkbox +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.FabPosition +import androidx.compose.material3.FloatingActionButton +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.TopAppBar +import androidx.compose.material3.TopAppBarDefaults +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.style.TextDecoration +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.hilt.navigation.compose.hiltViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.android.ai.samples.geminilivetodo.R +import com.android.ai.samples.geminilivetodo.data.Todo +import kotlin.collections.reversed + +/** + * The main screen for the To-do list application. + * This composable is stateful, connecting to the ViewModel to manage UI state and events. + */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun TodoScreen(viewModel: TodoScreenViewModel = hiltViewModel()) { + val uiState by viewModel.uiState.collectAsStateWithLifecycle() + var text by remember { mutableStateOf("") } + + val activity = LocalActivity.current as Activity + + LaunchedEffect(Unit) { + viewModel.initializeGeminiLive(activity) + } + + Scaffold( + topBar = { + TopAppBar( + colors = TopAppBarDefaults.topAppBarColors( + containerColor = MaterialTheme.colorScheme.primaryContainer, + titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, + ), + title = { Text(stringResource(R.string.gemini_live_title)) }, + ) + }, + floatingActionButton = { + MicButton( + uiState = uiState, + onToggle = { viewModel.toggleLiveSession(activity) }, + ) + }, + floatingActionButtonPosition = FabPosition.Center, + modifier = Modifier.fillMaxSize(), + ) { paddingValues -> + Column( + modifier = Modifier + .padding(paddingValues) + .padding(16.dp) + .imePadding() + .fillMaxSize(), + ) { + TodoInput( + text = text, + onTextChange = { text = it }, + onAddClick = { + viewModel.addTodo(text) + text = "" + }, + ) + + when (uiState) { + is TodoScreenUiState.Initial -> { + Box( + modifier = Modifier + .fillMaxSize(), + contentAlignment = Alignment.Center, + ) { + CircularProgressIndicator() + } + } + is TodoScreenUiState.Success -> { + val todos = (uiState as TodoScreenUiState.Success).todos + LazyColumn(modifier = Modifier.fillMaxSize()) { + itemsIndexed(todos.reversed(), key = { index: Int, item: Todo -> item.id }) { index, todo -> + TodoItem( + modifier = Modifier, + task = todo, + onToggle = { viewModel.toggleTodoStatus(todo.id) }, + onDelete = { viewModel.removeTodo(todo.id) }, + ) + if (index != todos.size - 1) { + HorizontalDivider() + } + } + } + } + is TodoScreenUiState.Error -> { + val todos = (uiState as TodoScreenUiState.Error).todos + LazyColumn(modifier = Modifier.fillMaxSize()) { + itemsIndexed(todos.reversed(), key = { index: Int, item: Todo -> item.id }) { index, todo -> + TodoItem( + modifier = Modifier, + task = todo, + onToggle = { viewModel.toggleTodoStatus(todo.id) }, + onDelete = { viewModel.removeTodo(todo.id) }, + ) + if (index != todos.size - 1) { + HorizontalDivider() + } + } + } + } + } + } + } +} + +@Composable +fun TodoInput(text: String, onTextChange: (String) -> Unit, onAddClick: () -> Unit) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(bottom = 16.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + OutlinedTextField( + value = text, + onValueChange = onTextChange, + label = { Text(stringResource(R.string.new_task_placeholder)) }, + modifier = Modifier.weight(1f), + singleLine = true, + ) + Spacer(modifier = Modifier.width(8.dp)) + Button( + enabled = text.isNotBlank(), + onClick = onAddClick, + ) { + Text(stringResource(R.string.add_button)) + } + } +} + +@Composable +fun MicButton(uiState: TodoScreenUiState, onToggle: () -> Unit) { + if (uiState is TodoScreenUiState.Success) { + val micIcon = when { + uiState.liveSessionState is LiveSessionState.Ready -> Icons.Filled.MicOff + uiState.liveSessionState is LiveSessionState.Running -> Icons.Filled.Mic + uiState.liveSessionState is LiveSessionState.NotReady -> Icons.Filled.MicNone + uiState.liveSessionState is LiveSessionState.Error -> Icons.Filled.MicNone + else -> Icons.Filled.MicNone + } + + val containerColor = if (uiState.liveSessionState is LiveSessionState.Running) { + val infiniteTransition = + rememberInfiniteTransition(label = "mic_color_transition") + infiniteTransition.animateColor( + initialValue = MaterialTheme.colorScheme.primary.copy(alpha = 0.2f), + targetValue = MaterialTheme.colorScheme.primary.copy(alpha = 0.9f), + animationSpec = infiniteRepeatable( + animation = tween(1000, easing = LinearEasing), + repeatMode = RepeatMode.Reverse, + ), + label = "mic_color", + ).value + } else { + MaterialTheme.colorScheme.primaryContainer + } + + FloatingActionButton( + onClick = { if (uiState.liveSessionState !is LiveSessionState.NotReady) onToggle() }, + containerColor = containerColor, + ) { + Icon(micIcon, stringResource(R.string.interact_with_todolist_by_voice)) + } + } else if (uiState is TodoScreenUiState.Error) { + val isDialogDisplayed = remember { mutableStateOf(true) } + if (isDialogDisplayed.value) { + AlertDialog( + onDismissRequest = { isDialogDisplayed.value = false }, + title = { Text(text = stringResource(R.string.error_title)) }, + text = { Text(text = stringResource(R.string.error_message)) }, + confirmButton = { + Button(onClick = { isDialogDisplayed.value = false }) { + Text(text = stringResource(R.string.dismiss_button)) + } + }, + ) + } + } +} + +@Composable +fun TodoItem(modifier: Modifier, task: Todo, onToggle: () -> Unit, onDelete: () -> Unit) { + val defaultBackgroundColor = Color.Transparent + val backgroundColor = remember { Animatable(defaultBackgroundColor) } + + Row( + modifier = Modifier + .fillMaxWidth() + .padding(vertical = 12.dp, horizontal = 8.dp) + .background(backgroundColor.value), + verticalAlignment = Alignment.CenterVertically, + ) { + Checkbox( + checked = task.isCompleted, + onCheckedChange = { onToggle() }, + ) + Text( + text = task.task, + style = if (task.isCompleted) { + TextStyle(fontSize = 16.sp, textDecoration = TextDecoration.LineThrough) + } else { + TextStyle(fontSize = 16.sp, textDecoration = TextDecoration.None) + }, + modifier = Modifier.weight(1f), + ) + IconButton(onClick = onDelete) { + Icon( + imageVector = Icons.Default.Delete, + contentDescription = "Delete", + ) + } + } +} diff --git a/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenUiState.kt b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenUiState.kt new file mode 100644 index 00000000..fc95618b --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenUiState.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ai.samples.geminilivetodo.ui + +import com.android.ai.samples.geminilivetodo.data.Todo + +sealed interface TodoScreenUiState { + data object Initial : TodoScreenUiState + + data class Success( + val todos: List = emptyList(), + val liveSessionState: LiveSessionState, + ) : TodoScreenUiState + + data class Error( + val todos: List = emptyList(), + val liveSessionState: LiveSessionState, + ) : TodoScreenUiState +} + +sealed interface LiveSessionState { + data object NotReady : LiveSessionState + data object Ready : LiveSessionState + data object Running : LiveSessionState + data object Error : LiveSessionState +} diff --git a/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt new file mode 100644 index 00000000..1e0e83bd --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt @@ -0,0 +1,246 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ai.samples.geminilivetodo.ui + +import android.Manifest +import android.annotation.SuppressLint +import android.app.Activity +import android.content.pm.PackageManager +import android.util.Log +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.android.ai.samples.geminilivetodo.data.TodoRepository +import com.google.firebase.Firebase +import com.google.firebase.ai.ai +import com.google.firebase.ai.type.FunctionCallPart +import com.google.firebase.ai.type.FunctionDeclaration +import com.google.firebase.ai.type.FunctionResponsePart +import com.google.firebase.ai.type.GenerativeBackend +import com.google.firebase.ai.type.LiveSession +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.ResponseModality +import com.google.firebase.ai.type.Schema +import com.google.firebase.ai.type.SpeechConfig +import com.google.firebase.ai.type.Tool +import com.google.firebase.ai.type.Voice +import com.google.firebase.ai.type.content +import com.google.firebase.ai.type.liveGenerationConfig +import dagger.hilt.android.lifecycle.HiltViewModel +import javax.inject.Inject +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long + +@OptIn(PublicPreviewAPI::class) +@HiltViewModel +class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRepository) : ViewModel() { + private val TAG = "TodoScreenViewModel" + private var session: LiveSession? = null + + private val liveSessionState = MutableStateFlow(LiveSessionState.NotReady) + private val todos = todoRepository.todos + + val uiState: StateFlow = combine(liveSessionState, todos) { liveSessionState, todos -> + TodoScreenUiState.Success(todos, liveSessionState) + }.stateIn( + scope = viewModelScope, + started = SharingStarted.WhileSubscribed(5000L), + initialValue = TodoScreenUiState.Initial, + ) + + fun addTodo(taskDescription: String) { + todoRepository.addTodo(taskDescription) + } + + fun removeTodo(todoId: Long) { + todoRepository.removeTodo(todoId) + } + + fun toggleTodoStatus(todoId: Long) { + todoRepository.toggleTodoStatus(todoId) + } + + @SuppressLint("MissingPermission") + fun toggleLiveSession(activity: Activity) { + viewModelScope.launch { + if (liveSessionState.value is LiveSessionState.NotReady) return@launch + + session?.let { + if (liveSessionState.value is LiveSessionState.Ready) { + if (ContextCompat.checkSelfPermission( + activity, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + ) { + it.startAudioConversation(::handleFunctionCall) + liveSessionState.value = LiveSessionState.Running + } + } else { + it.stopAudioConversation() + liveSessionState.value = LiveSessionState.Ready + } + } + } + } + + fun initializeGeminiLive(activity: Activity) { + requestAudioPermissionIfNeeded(activity) + viewModelScope.launch { + Log.d(TAG, "Start Gemini Live initialization") + val liveGenerationConfig = liveGenerationConfig { + speechConfig = SpeechConfig(voice = Voice("FENRIR")) + responseModality = ResponseModality.AUDIO + } + + val systemInstruction = content { + text( + """ + **Your Role:** You are a friendly and helpful voice assistant in this app. + Your main job is to change update the tasks in the todo list based on user requests. + + **Interaction Steps:** + **Get the task id to remove or toggle a task:** If you need to remove or check/uncheck a task, + you'll need to retrieve the list of items in the list first to get the task id. Don't share + the id with the user, just identify the id of the task mentioned and directly pass this id to the + tool. + + **Never share the id with the user:** you don't need to share the id with the user. It is + just here to help you perform the check/uncheck and remove operations to the list. + + **If Unsure:** If you can't determine the update from the request, politely ask the user to rephrase or try something else. + """.trimIndent(), + ) + } + + val addTodo = FunctionDeclaration( + "addTodo", + "Add a task to the todo list", + mapOf("taskDescription" to Schema.string("A succinct string describing the task")), + ) + + val removeTodo = FunctionDeclaration( + "removeTodo", + "Remove a task from the todo list", + mapOf("todoId" to Schema.string("The id of the task to remove from the todo list")), + ) + + val toggleTodoStatus = FunctionDeclaration( + "toggleTodoStatus", + "Change the status of the task", + mapOf("todoId" to Schema.string("The id of the task to remove from the todo list")), + ) + + val getTodoList = FunctionDeclaration( + "getTodoList", + "Get the list of all the tasks in the todo list", + emptyMap(), + ) + + val generativeModel = Firebase.ai(backend = GenerativeBackend.vertexAI()).liveModel( + "gemini-2.0-flash-live-preview-04-09", + generationConfig = liveGenerationConfig, + systemInstruction = systemInstruction, + tools = listOf( + Tool.functionDeclarations( + listOf(getTodoList, addTodo, removeTodo, toggleTodoStatus), + ), + ), + ) + + try { + session = generativeModel.connect() + } catch (e: Exception) { + Log.e(TAG, "Error connecting to the model", e) + liveSessionState.value = LiveSessionState.Error + } + + liveSessionState.value = LiveSessionState.Ready + } + } + + private fun handleFunctionCall(functionCall: FunctionCallPart): FunctionResponsePart { + return when (functionCall.name) { + "getTodoList" -> { + val todoList = todoRepository.getTodoList().reversed() + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("List of tasks in the todo list: $todoList"), + ), + ) + FunctionResponsePart(functionCall.name, response) + } + "addTodo" -> { + val taskDescription = functionCall.args["taskDescription"]!!.jsonPrimitive.content + todoRepository.addTodo(taskDescription) + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task $taskDescription added to the todo list"), + ), + ) + FunctionResponsePart(functionCall.name, response) + } + "removeTodo" -> { + val taskId = functionCall.args["todoId"]!!.jsonPrimitive.long + todoRepository.removeTodo(taskId) + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task was removed from the todo list"), + ), + ) + FunctionResponsePart(functionCall.name, response) + } + "toggleTodoStatus" -> { + val taskId = functionCall.args["todoId"]!!.jsonPrimitive.long + todoRepository.toggleTodoStatus(taskId) + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task was toggled in the todo list"), + ), + ) + FunctionResponsePart(functionCall.name, response) + } + else -> { + val response = JsonObject( + mapOf("error" to JsonPrimitive("Unknown function: ${functionCall.name}")), + ) + FunctionResponsePart(functionCall.name, response) + } + } + } + + fun requestAudioPermissionIfNeeded(activity: Activity) { + if (ContextCompat.checkSelfPermission( + activity, + Manifest.permission.RECORD_AUDIO, + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions(activity, arrayOf(Manifest.permission.RECORD_AUDIO), 1) + } + } +} diff --git a/ai-catalog/samples/gemini-live-todo/src/main/res/values/strings.xml b/ai-catalog/samples/gemini-live-todo/src/main/res/values/strings.xml new file mode 100644 index 00000000..60932bb7 --- /dev/null +++ b/ai-catalog/samples/gemini-live-todo/src/main/res/values/strings.xml @@ -0,0 +1,26 @@ + + + + Gemini Live Todo + New Task + Add + Error + The live session model could not be initialized. + Dismiss + Button to start the live session and interact with the todo list by voice + \ No newline at end of file diff --git a/ai-catalog/settings.gradle.kts b/ai-catalog/settings.gradle.kts index 9fa0bfea..6ba4d0ff 100644 --- a/ai-catalog/settings.gradle.kts +++ b/ai-catalog/settings.gradle.kts @@ -46,3 +46,4 @@ include(":samples:genai-image-description") include(":samples:imagen") include(":samples:magic-selfie") include(":samples:gemini-video-summarization") +include(":samples:gemini-live-todo")