Merge pull request #170 from android/github_live

Use Nano to generate image description

Author: calren

app/src/main/AndroidManifest.xml

@@ -1,5 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
+<?xml version="1.0" encoding="utf-8"?><!--
      Copyright 2025 The Android Open Source Project
 
      Licensed under the Apache License, Version 2.0 (the "License");
@@ -24,10 +23,11 @@
         android:required="true" />
 
     <!-- Androidify can use XR features if they're available; they're not required. -->
-    <uses-feature android:name="android.software.xr.api.spatial" android:required="false" />
+    <uses-feature
+        android:name="android.software.xr.api.spatial"
+        android:required="false" />
 
     <uses-permission android:name="android.permission.CAMERA" />
-    <uses-sdk tools:overrideLibrary="com.google.ai.edge.aicore" />
 
     <application
         android:name=".AndroidifyApplication"
@@ -68,6 +68,9 @@
             <meta-data
                 android:name="com.android.developers.androidify.startup.FirebaseRemoteConfigInitializer"
                 android:value="@string/androidx_startup" />
+            <meta-data
+                android:name="com.android.developers.androidify.startup.GeminiNanoDownloaderInitializer"
+                android:value="@string/androidx_startup" />
         </provider>
 
         <activity
@@ -84,10 +87,13 @@
             <!-- Required deeplink to make the app launchable from the watch -->
             <intent-filter>
                 <action android:name="android.intent.action.VIEW" />
+
                 <category android:name="android.intent.category.DEFAULT" />
                 <category android:name="android.intent.category.BROWSABLE" />
-                <data android:scheme="androidify"
-                    android:host="launch" />
+
+                <data
+                    android:host="launch"
+                    android:scheme="androidify" />
             </intent-filter>
         </activity>
         <!-- need to use Theme.AppCompat -->
@@ -105,4 +111,4 @@
             android:value="subject_segmentation" />
     </application>
 
-</manifest>
+</manifest>

core/network/src/main/java/com/android/developers/androidify/RemoteConfigDataSource.kt

@@ -27,8 +27,11 @@ interface RemoteConfigDataSource {
     fun isBackgroundVibesFeatureEnabled(): Boolean
     fun promptTextVerify(): String
     fun promptImageValidation(): String
+    fun promptImageValidationNano(): String
     fun promptImageDescription(): String
+    fun promptImageDescriptionNano(): String
     fun useGeminiNano(): Boolean
+    fun enabledGeminiNanoModelVersions(): String
     fun generateBotPrompt(): String
     fun promptImageGenerationWithSkinTone(): String
 
@@ -77,14 +80,26 @@ class RemoteConfigDataSourceImpl @Inject constructor() : RemoteConfigDataSource
         return remoteConfig.getString("prompt_image_validation")
     }
 
+    override fun promptImageValidationNano(): String {
+        return remoteConfig.getString("prompt_image_validation_nano")
+    }
+
     override fun promptImageDescription(): String {
         return remoteConfig.getString("prompt_image_description")
     }
 
+    override fun promptImageDescriptionNano(): String {
+        return remoteConfig.getString("prompt_image_description_nano")
+    }
+
     override fun useGeminiNano(): Boolean {
         return remoteConfig.getBoolean("use_gemini_nano")
     }
 
+    override fun enabledGeminiNanoModelVersions(): String {
+        return remoteConfig.getString("enabled_gemini_nano_model_versions")
+    }
+
     override fun generateBotPrompt(): String {
         return remoteConfig.getString("generate_bot_prompt")
     }

core/network/src/main/res/xml/remote_config_defaults.xml

@@ -62,10 +62,36 @@
             - it cannot contain  hate speech or other offensive language
             -it cannot contain blood or gore or violence.</value>
     </entry>
+    <entry>
+        <key>prompt_image_validation_nano</key>
+        <value>[TASK]
+            You are a Validator. Analyze the attached image and determine its validity based on the
+            rules.
+            [RULES]
+            VALID if AND ONLY if:
+            1. PRIMARY subject is a person showing their head and shoulder
+            2. The image MUST NOT contain: Nudity, Explicit content, Illegal weapons, Violent
+            references, Drugs, Illicit substances, Hate speech, Offensive language, Blood, Gore, or
+            Violence.
+            [OUTPUT]
+            Return ONLY one string.
+            Check sequentially. Output the first failure code that applies:
+            1. Is the PRIMARY subject NOT a person (e.g., animal, object, landscape)? ->
+            "not_a_person"
+            2. Is the person present but missing face/head/shoulders or too blurry? ->
+            "not_enough_detail"
+            3. Does the image violate any negative policy (Rule 2)? -> "policy_violation"
+            4. If all rules are passed: -> null
+        </value>
+    </entry>
     <entry>
         <key>use_gemini_nano</key>
         <value>false</value>
     </entry>
+    <entry>
+        <key>enabled_gemini_nano_model_versions</key>
+        <value>nano-v3</value>
+    </entry>
     <entry>
         <key>dancing_droid_gif_link</key>
         <value>https://services.google.com/fh/files/misc/android_dancing.gif</value>
@@ -148,6 +174,45 @@
             * Do not say rendered, rendering, or digital.
             * Only respond with new image description as a paragraph.</value>
     </entry>
+    <entry>
+        <key>prompt_image_description_nano</key>
+        <value>## Role
+            You are an expert image analyst specializing in generating detailed, objective descriptions of people.
+
+            ## Task
+            Your task is to describe the person in the provided image in vivid detail, following the guidelines and examples below.
+
+            ## Guidelines
+            - Start with the overall mood or impression of the person (e.g., serene, joyful, pensive).
+            - Describe the person's physical appearance, focusing on hair (color, style, length) and any visible facial features.
+            - Detail the clothing, including the type of garments, style, color, and material.
+            - Mention any accessories, such as glasses, hats, or jewelry.
+            - Describe the immediate surroundings, including any objects, animals, or other people interacting with the subject.
+
+            ## Constraints
+            - The output must be a single, coherent paragraph.
+            - If no person is visible in the image, state that clearly and do not describe anything else.
+            - Provide only the description. Do not add any introductory or concluding remarks.
+
+            ## Examples
+
+            ### Example 1: Standard Case
+            Input: [Image of a person on a picnic blanket with a dog]
+            Output: A highly detailed and realistic portrayal of a person with a serene and pleasant mood. The figure has short, chin-length, straight dark black hair. No facial hair is present. Blue mirrored sunglasses are resting on top of its head. The figure is wearing a loose-fitting, light gray kimono-like top with a V-neckline and wide, elbow-length sleeves. This top features intricate, colorful embroidery in muted red, green, and yellow floral patterns on the front and sleeves. On its bottom, the figure wears loose-fitting, light gray wide-leg pants made of a soft, flowing material. No footwear is visible. The figure is seated on a red and white checkered picnic blanket. Next to it on the blanket is a clear plastic bottle. It is interacting with a black and white Pomeranian-like dog, which has black fur with distinct white markings on its chest, legs, and face, and a leash attached to its collar. The overall depiction aims for a clear and life-like appearance.
+
+            ### Example 2: Corner Case (No Person)
+            Input: [Image of an empty park bench]
+            Output: No person is visible in the image.
+
+            ## Input
+            {{image}}
+
+            ## Output Reminder
+            Take a deep breath, read the instructions again, read the inputs again. Each instruction is crucial and must be executed with utmost care and attention to detail.
+
+            Description:
+        </value>
+    </entry>
     <entry>
         <key>promo_video_link</key>
         <value>https://services.google.com/fh/files/misc/androidfy_storyboard_b_v07.mp4</value>

core/testing/src/main/AndroidManifest.xml

@@ -17,5 +17,4 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:tools="http://schemas.android.com/tools">
 
-    <uses-sdk tools:overrideLibrary="com.google.ai.edge.aicore"/>
 </manifest>

core/testing/src/main/java/com/android/developers/testing/data/TestGeminiNanoDownloader.kt

@@ -0,0 +1,9 @@
+package com.android.developers.testing.data
+
+import com.android.developers.androidify.data.GeminiNanoDownloader
+
+class TestGeminiNanoDownloader(val modelDownloaded: Boolean) : GeminiNanoDownloader {
+    override fun isModelDownloaded(): Boolean {
+        return modelDownloaded
+    }
+}

core/testing/src/main/java/com/android/developers/testing/data/TestGeminiNanoGenerationDataSource.kt

@@ -15,13 +15,27 @@
  */
 package com.android.developers.testing.data
 
+import android.graphics.Bitmap
+import com.android.developers.androidify.data.GeminiNanoDownloader
 import com.android.developers.androidify.data.GeminiNanoGenerationDataSource
+import com.android.developers.androidify.model.ValidatedDescription
+import com.android.developers.androidify.model.ValidatedImage
 
-class TestGeminiNanoGenerationDataSource(val promptOutput: String?) : GeminiNanoGenerationDataSource {
-    override suspend fun initialize() {
-    }
+class TestGeminiNanoGenerationDataSource(
+    val promptOutput: String?,
+    val geminiNanoDownloader: GeminiNanoDownloader
+) : GeminiNanoGenerationDataSource {
 
     override suspend fun generatePrompt(prompt: String): String? {
         return promptOutput
     }
+
+    override suspend fun validateImageHasEnoughInformation(image: Bitmap): ValidatedImage? {
+        return ValidatedImage(true, null)
+    }
+
+    override suspend fun generateDescriptivePromptFromImage(image: Bitmap): ValidatedDescription? {
+        if (!geminiNanoDownloader.isModelDownloaded()) return null
+        return ValidatedDescription(true, "Nano description")
+    }
 }

core/testing/src/main/java/com/android/developers/testing/network/TestFirebaseAiDataSource.kt

@@ -24,15 +24,15 @@ import com.android.developers.androidify.vertexai.FirebaseAiDataSource
 
 class TestFirebaseAiDataSource(val promptOutput: List<String>) : FirebaseAiDataSource {
     override suspend fun validatePromptHasEnoughInformation(inputPrompt: String): ValidatedDescription {
-        return ValidatedDescription(true, "User description")
+        return ValidatedDescription(true, "Firebase description")
     }
 
     override suspend fun validateImageHasEnoughInformation(image: Bitmap): ValidatedImage {
         return ValidatedImage(true, null)
     }
 
     override suspend fun generateDescriptivePromptFromImage(image: Bitmap): ValidatedDescription {
-        return ValidatedDescription(true, "User description")
+        return ValidatedDescription(true, "Firebase description")
     }
 
     override suspend fun generateImageFromPromptAndSkinTone(

core/testing/src/main/java/com/android/developers/testing/network/TestLocalSegmentationDataSource.kt

@@ -0,0 +1,12 @@
+package com.android.developers.testing.network
+
+import android.graphics.Bitmap
+import com.android.developers.androidify.ondevice.LocalSegmentationDataSource
+import androidx.core.graphics.createBitmap
+
+class TestLocalSegmentationDataSource() : LocalSegmentationDataSource {
+
+    override suspend fun removeBackground(bitmap: Bitmap): Bitmap {
+        return createBitmap(100, 100)
+    }
+}

core/testing/src/main/java/com/android/developers/testing/network/TestRemoteConfigDataSource.kt

@@ -42,14 +42,26 @@ class TestRemoteConfigDataSource(private val useGeminiNano: Boolean) : RemoteCon
         TODO("Not yet implemented")
     }
 
+    override fun promptImageValidationNano(): String {
+        TODO("Not yet implemented")
+    }
+
     override fun promptImageDescription(): String {
         TODO("Not yet implemented")
     }
 
+    override fun promptImageDescriptionNano(): String {
+        TODO("Not yet implemented")
+    }
+
     override fun useGeminiNano(): Boolean {
         return useGeminiNano
     }
 
+    override fun enabledGeminiNanoModelVersions(): String {
+        TODO("Not yet implemented")
+    }
+
     override fun generateBotPrompt(): String {
         return "generateBotPrompt"
     }

core/testing/src/main/java/com/android/developers/testing/repository/FakeImageGenerationRepository.kt

@@ -20,11 +20,10 @@ import android.net.Uri
 import androidx.core.graphics.createBitmap
 import androidx.core.net.toUri
 import com.android.developers.androidify.data.ImageGenerationRepository
+import com.android.developers.androidify.model.ValidatedDescription
 import java.io.File
 
 class FakeImageGenerationRepository : ImageGenerationRepository {
-    override suspend fun initialize() {
-    }
     var exceptionToThrow: Exception? = null
 
     override suspend fun generateFromDescription(
@@ -35,6 +34,10 @@ class FakeImageGenerationRepository : ImageGenerationRepository {
         return createBitmap(1, 1)
     }
 
+    override suspend fun getDescriptionFromImage(file: File): ValidatedDescription {
+        return ValidatedDescription(true, "")
+    }
+
     override suspend fun generateFromImage(
         file: File,
         skinTone: String,