[ALF] Add LiveAPI video sample

firebase-ai/app/build.gradle.kts

@@ -67,6 +67,13 @@ dependencies {
     // Webkit
     implementation(libs.androidx.webkit)
 
+    // CameraX
+    implementation(libs.androidx.camera.core)
+    implementation(libs.androidx.camera.camera2)
+    implementation(libs.androidx.camera.lifecycle)
+    implementation(libs.androidx.camera.view)
+    implementation(libs.androidx.camera.extensions)
+
     // Material for XML-based theme
     implementation(libs.material)
 

firebase-ai/app/src/main/AndroidManifest.xml

@@ -4,8 +4,12 @@
 
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
-    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    <uses-permission android:name="android.permission.CAMERA" />
+
+    <uses-feature android:name="android.hardware.camera" />
+    <uses-feature android:name="android.hardware.microphone" />
+
     <application
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"
@@ -19,7 +23,7 @@
         <activity
             android:name=".MainActivity"
             android:exported="true"
-            android:label="@string/app_name"
+
             android:theme="@style/Theme.FirebaseAIServices">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />

firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/FirebaseAISamples.kt

@@ -275,6 +275,8 @@ val FIREBASE_AI_SAMPLES = listOf(
         description = "Use bidirectional streaming to get information about" +
                 " weather conditions for a specific US city on a specific date",
         navRoute = "stream",
+        backend = GenerativeBackend.vertexAI(),
+        modelName = "gemini-2.0-flash-live-preview-04-09",
         categories = listOf(Category.LIVE_API, Category.AUDIO, Category.FUNCTION_CALLING),
         tools = listOf(
             Tool.functionDeclarations(
@@ -298,6 +300,36 @@ val FIREBASE_AI_SAMPLES = listOf(
             text("What was the weather in Boston, MA on October 17, 2024?")
         }
     ),
+    Sample(
+        title = "Video input",
+        description = "Use bidirectional streaming to chat with Gemini using your" +
+                " phone's camera",
+        navRoute = "streamVideo",
+        backend = GenerativeBackend.vertexAI(),
+        modelName = "gemini-2.0-flash-live-preview-04-09",
+        categories = listOf(Category.LIVE_API, Category.VIDEO, Category.FUNCTION_CALLING),
+        tools = listOf(
+            Tool.functionDeclarations(
+                listOf(
+                    FunctionDeclaration(
+                        "fetchWeather",
+                        "Get the weather conditions for a specific US city on a specific date.",
+                        mapOf(
+                            "city" to Schema.string("The US city of the location."),
+                            "state" to Schema.string("The US state of the location."),
+                            "date" to Schema.string(
+                                "The date for which to get the weather." +
+                                        " Date must be in the format: YYYY-MM-DD."
+                            ),
+                        ),
+                    )
+                )
+            )
+        ),
+        initialPrompt = content {
+            text("What was the weather in Boston, MA on October 17, 2024?")
+        }
+    ),
     Sample(
         title = "Weather Chat",
         description = "Use function calling to get the weather conditions" +

firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt

@@ -1,6 +1,7 @@
 package com.google.firebase.quickstart.ai
 
 import android.Manifest
+import android.annotation.SuppressLint
 import android.content.pm.PackageManager
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
@@ -31,6 +32,8 @@ import androidx.navigation.compose.rememberNavController
 import com.google.firebase.ai.type.toImagenInlineImage
 import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeRoute
 import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeScreen
+import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeVideoRoute
+import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeVideoScreen
 import com.google.firebase.quickstart.ai.feature.media.imagen.ImagenRoute
 import com.google.firebase.quickstart.ai.feature.media.imagen.ImagenScreen
 import com.google.firebase.quickstart.ai.feature.text.ChatRoute
@@ -42,10 +45,7 @@ class MainActivity : ComponentActivity() {
     @OptIn(ExperimentalMaterial3Api::class)
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        if(ContextCompat.checkSelfPermission(this,
-                Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
-            ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.RECORD_AUDIO), 1)
-        }
+
         enableEdgeToEdge()
         catImage = BitmapFactory.decodeResource(applicationContext.resources, R.drawable.cat)
         setContent {
@@ -90,6 +90,9 @@ class MainActivity : ComponentActivity() {
                                         "stream" -> {
                                             navController.navigate(StreamRealtimeRoute(it.id))
                                         }
+                                        "streamVideo" -> {
+                                            navController.navigate(StreamRealtimeVideoRoute(it.id))
+                                        }
                                     }
                                 }
                             )
@@ -102,10 +105,18 @@ class MainActivity : ComponentActivity() {
                         composable<ImagenRoute> {
                             ImagenScreen()
                         }
-                        // Stream Realtime Samples
+                        // The permission is checked by the @RequiresPermission annotation on the
+                        // StreamRealtimeScreen composable.
+                        @SuppressLint("MissingPermission")
                         composable<StreamRealtimeRoute> {
                             StreamRealtimeScreen()
                         }
+                        // The permission is checked by the @RequiresPermission annotation on the
+                        // StreamRealtimeVideoScreen composable.
+                        @SuppressLint("MissingPermission")
+                        composable<StreamRealtimeVideoRoute> {
+                            StreamRealtimeVideoScreen()
+                        }
                     }
                 }
             }

firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/BidiViewModel.kt

@@ -1,56 +1,33 @@
-package com.google.firebase.quickstart.ai.feature.media.imagen
+package com.google.firebase.quickstart.ai.feature.live
 
-import android.Manifest
+import android.annotation.SuppressLint
 import android.graphics.Bitmap
-import androidx.annotation.RequiresPermission
 import androidx.lifecycle.SavedStateHandle
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import androidx.navigation.toRoute
 import com.google.firebase.Firebase
 import com.google.firebase.ai.FirebaseAI
-import com.google.firebase.ai.ImagenModel
-import com.google.firebase.ai.LiveGenerativeModel
-import com.google.firebase.ai.ai
 import com.google.firebase.ai.type.FunctionCallPart
 import com.google.firebase.ai.type.FunctionResponsePart
-import com.google.firebase.ai.type.GenerativeBackend
-import com.google.firebase.ai.type.ImagenAspectRatio
-import com.google.firebase.ai.type.ImagenImageFormat
-import com.google.firebase.ai.type.ImagenPersonFilterLevel
-import com.google.firebase.ai.type.ImagenSafetyFilterLevel
-import com.google.firebase.ai.type.ImagenSafetySettings
-import com.google.firebase.ai.type.InlineDataPart
-import com.google.firebase.ai.type.LiveServerContent
-import com.google.firebase.ai.type.LiveServerMessage
+import com.google.firebase.ai.type.InlineData
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.PublicPreviewAPI
 import com.google.firebase.ai.type.ResponseModality
 import com.google.firebase.ai.type.SpeechConfig
-import com.google.firebase.ai.type.TextPart
-import com.google.firebase.ai.type.Tool
 import com.google.firebase.ai.type.Voice
-import com.google.firebase.ai.type.asTextOrNull
-import com.google.firebase.ai.type.imagenGenerationConfig
 import com.google.firebase.ai.type.liveGenerationConfig
 import com.google.firebase.app
 import com.google.firebase.quickstart.ai.FIREBASE_AI_SAMPLES
-import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeRoute
-import com.google.firebase.quickstart.ai.feature.text.functioncalling.WeatherRepository
 import com.google.firebase.quickstart.ai.feature.text.functioncalling.WeatherRepository.Companion.fetchWeather
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.flow.MutableStateFlow
-import kotlinx.coroutines.flow.StateFlow
+import java.io.ByteArrayOutputStream
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.jsonPrimitive
 
 @OptIn(PublicPreviewAPI::class)
-class BidiViewModel(
-    savedStateHandle: SavedStateHandle
-) : ViewModel() {
+class BidiViewModel(savedStateHandle: SavedStateHandle) : ViewModel() {
     private val sampleId = savedStateHandle.toRoute<StreamRealtimeRoute>().sampleId
     private val sample = FIREBASE_AI_SAMPLES.first { it.id == sampleId }
 
@@ -63,41 +40,54 @@ class BidiViewModel(
             // Change this to ContentModality.TEXT if you want text output.
             responseModality = ResponseModality.AUDIO
         }
+
         @OptIn(PublicPreviewAPI::class)
-        val liveModel = FirebaseAI.getInstance(Firebase.app, sample.backend).liveModel(
-            "gemini-live-2.5-flash",
-            generationConfig = liveGenerationConfig,
-            tools = sample.tools
-        )
-        runBlocking {
-            liveSession = liveModel.connect()
-        }
+        val liveModel =
+            FirebaseAI.getInstance(Firebase.app, sample.backend)
+                .liveModel(
+                    modelName = sample.modelName ?: "gemini-live-2.5-flash",
+                    generationConfig = liveGenerationConfig,
+                    tools = sample.tools,
+                )
+        runBlocking { liveSession = liveModel.connect() }
     }
 
-    fun handler(fetchWeatherCall: FunctionCallPart) : FunctionResponsePart {
-        val response:JsonObject
+    fun handler(fetchWeatherCall: FunctionCallPart): FunctionResponsePart {
+        val response: JsonObject
         fetchWeatherCall.let {
             val city = it.args["city"]?.jsonPrimitive?.content
             val state = it.args["state"]?.jsonPrimitive?.content
             val date = it.args["date"]?.jsonPrimitive?.content
             runBlocking {
-                response = if(!city.isNullOrEmpty() and !state.isNullOrEmpty() and date.isNullOrEmpty()) {
-                    fetchWeather(city!!, state!!, date!!)
-                } else {
-                    JsonObject(emptyMap())
-                }
+                response =
+                    if (!city.isNullOrEmpty() and !state.isNullOrEmpty() and date.isNullOrEmpty()) {
+                        fetchWeather(city!!, state!!, date!!)
+                    } else {
+                        JsonObject(emptyMap())
+                    }
             }
         }
-        return FunctionResponsePart("fetchWeather",  response, fetchWeatherCall.id)
+        return FunctionResponsePart("fetchWeather", response, fetchWeatherCall.id)
     }
-    @RequiresPermission(Manifest.permission.RECORD_AUDIO)
+
+    // The permission check is handled by the view that calls this function.
+    @SuppressLint("MissingPermission")
     suspend fun startConversation() {
-      liveSession.startAudioConversation(::handler)
+        liveSession.startAudioConversation(::handler)
     }
 
     fun endConversation() {
         liveSession.stopAudioConversation()
     }
 
+    fun sendVideoFrame(frame: Bitmap) {
+        viewModelScope.launch {
+            // Directly compress the Bitmap to a ByteArray
+            val byteArrayOutputStream = ByteArrayOutputStream()
+            frame.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream)
+            val jpegBytes = byteArrayOutputStream.toByteArray()
 
+            liveSession.sendVideoRealtime(InlineData(jpegBytes, "image/jpeg"))
+        }
+    }
 }

firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/CameraView.kt

@@ -0,0 +1,98 @@
+package com.google.firebase.quickstart.ai.feature.live
+
+import android.annotation.SuppressLint
+import android.graphics.Bitmap
+import androidx.camera.core.CameraSelector
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
+import androidx.camera.core.Preview
+import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.camera.view.PreviewView
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.remember
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalLifecycleOwner
+import androidx.compose.ui.viewinterop.AndroidView
+import androidx.core.content.ContextCompat
+import androidx.lifecycle.LifecycleOwner
+import kotlin.time.Duration.Companion.seconds
+
+@Composable
+fun CameraView(
+    modifier: Modifier = Modifier,
+    cameraSelector: CameraSelector = CameraSelector.DEFAULT_BACK_CAMERA,
+    onFrameCaptured: (Bitmap) -> Unit,
+) {
+    val context = LocalContext.current
+    val lifecycleOwner = LocalLifecycleOwner.current
+    val cameraProviderFuture = remember { ProcessCameraProvider.getInstance(context) }
+
+    AndroidView(
+        factory = { ctx ->
+            val previewView = PreviewView(ctx)
+            val executor = ContextCompat.getMainExecutor(ctx)
+            cameraProviderFuture.addListener(
+                {
+                    val cameraProvider = cameraProviderFuture.get()
+                    bindPreview(
+                        lifecycleOwner,
+                        previewView,
+                        cameraProvider,
+                        cameraSelector,
+                        onFrameCaptured,
+                    )
+                },
+                executor,
+            )
+            previewView
+        },
+        modifier = modifier,
+    )
+}
+
+private fun bindPreview(
+    lifecycleOwner: LifecycleOwner,
+    previewView: PreviewView,
+    cameraProvider: ProcessCameraProvider,
+    cameraSelector: CameraSelector,
+    onFrameCaptured: (Bitmap) -> Unit,
+) {
+    val preview =
+        Preview.Builder().build().also { it.setSurfaceProvider(previewView.surfaceProvider) }
+
+    val imageAnalysis =
+        ImageAnalysis.Builder()
+            .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+            .build()
+            .also {
+                it.setAnalyzer(
+                    ContextCompat.getMainExecutor(previewView.context),
+                    SnapshotFrameAnalyzer(onFrameCaptured),
+                )
+            }
+
+    cameraProvider.unbindAll()
+    cameraProvider.bindToLifecycle(lifecycleOwner, cameraSelector, preview, imageAnalysis)
+}
+
+// Calls the [onFrameCaptured] callback with the captured frame every second.
+private class SnapshotFrameAnalyzer(private val onFrameCaptured: (Bitmap) -> Unit) :
+    ImageAnalysis.Analyzer {
+    private var lastFrameTimestamp = 0L
+    private val interval = 1.seconds // 1 second
+
+    @SuppressLint("UnsafeOptInUsageError")
+    override fun analyze(image: ImageProxy) {
+        val currentTimestamp = System.currentTimeMillis()
+        if (lastFrameTimestamp == 0L) {
+            lastFrameTimestamp = currentTimestamp
+        }
+
+        if (currentTimestamp - lastFrameTimestamp >= interval.inWholeMilliseconds) {
+            onFrameCaptured(image.toBitmap())
+            lastFrameTimestamp = currentTimestamp
+        }
+        image.close()
+    }
+}

firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamRealtimeScreen.kt

@@ -32,8 +32,6 @@ import androidx.compose.runtime.remember
 import androidx.compose.runtime.mutableStateOf
 
 import androidx.lifecycle.viewmodel.compose.viewModel
-import com.google.firebase.quickstart.ai.feature.media.imagen.BidiViewModel
-import com.google.firebase.quickstart.ai.feature.media.imagen.ImagenViewModel
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch