Genai speech to form (#3666)

* Initial commit

* Add dependencies

* Transcribe an audio file to text using Google Cloud Speech-to-Text API

* Gemini client

* TextToForm code

* remove validation

* SpeechToForm class

* add genai lib, mock gemini class with it

* use suspend, rename to model, update template

* Add unit tests

* Add lisences, clean up

* fix tests

* expand docs

* Allow for GEMINI_API_KEY to be passed as project property

* Update SpeechToForm to use GeminiModel in instantiating TextToForm
Update GeminiModel to return model

* Clean up temp file after test

* Add option to run tests bu hitting the actual API

* Update method name to avoid conflict

* pull out qr validation

* QR generation retry logic incase of errors.
Refactor some functions

* Address changes

* Add test audio file

* move to new package structure and abstract models

* match tests to new abstractions

* turn class into interface, do not override autogen getter

* retry if not able to extract response, always return a qr

* refactor some constants

* Add sample transcript

* rename packages and add outline of summarize class

* add scaffold for live speech to text

* Refine model layer abstraction.
Make it easy to change models.

* Refactor dispatcher.

* Convert Speech to text to an object.
Use proper logging.
Remove redundant qR checks

* refactor TextToForm to an object. Utilize refactored models.
Handle different responses

* Update tests

* fix summarize and remove wildcard

* spotless

* Resolved issues making build fail:
- Excluded conflicting dependencies (com.google.guava:guava and org.threeten:threetenbp) from the google-cloud-speech library.
- Upgraded google-cloud-speech to version 4.50.0 to address compatibility issues.

* Additional sample test files

* Use real files TextToForm tests

* Add grpc-okhttp dependency needed by google cloud speech

* Switch transcription to only transcribe wav files as recommended.

* Improve SpeechToTextTest to only mock api calls but use real files

* Add sample conversation.wav file

* Handle edge case in retryLogic where QR, QRJson of generatedText could be blank
Add more TextToFormTest tests

* Gnerate Summary initial work and tests

* Resolve tests failure caused by refactoring QuestionnaireResponseValidator

* Add SpeechToext fragment views

* add starter gemma model

* Update questionnaire form on new QuestionnaireResponse

* Refactor showing progress dialog in Questionnaire activity

* Show record button

* Toggle record button and show speech-to-text 'window'

* Exclude protobuf-javalite to fix duplicated classes error

Since full version protobuf-java is required for com.google.cloud:google-cloud-speech

* Add record-audio permission before lauching speech to text

* Simulate stop recording processing progress view

* add tests, fix style

* set type and create constants

* Integrate to TextToForm api with Gemini

* Show edit button to exit speech-to-text without updating form

* Integrate speech-to-text to call cloud speech-to-text api

* Fix errors in tests

* Fix failing tests for QuestionnaireViewModel

* Latest working changes - add build variant

* Run spotless

* Add enableSpeechToText to questionnaire config

* Edit GemmaModel class to allow test

* refactor hasDebygSuffix and allow more paths

* check buffer min and use generic mic

* newer model, fix prompt for multiselects

* line breaks

* move hasDebugSuffix tests

* use a list and remove need for test

* extend timeout, check adb

* add ram size

* add missing mocks and ids

* add test for service points map

* test unmocked debug variant

* add note for gradle change

* increase heap

* disable avd force

* remove memory changes

* shard quest tests

* fix time stripping code

* add info

* add shard suffix to test report

* remove clean

* update regex and examples

* add worker cleanup

* add unmock

* more agressive test cleanup

* remove verbose

* add roll up step

* increase heap size

* more shards less heap

---------

Co-authored-by: pld <peter@ona.io>
Co-authored-by: L≡ZRS <12814349+LZRS@users.noreply.github.com>
Co-authored-by: Francis Odhiambo <4540684+f-odhiambo@users.noreply.github.com>

Author: 4 people

.github/workflows/ci.yml

@@ -25,11 +25,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        api-level: [34]
-        
+        api-level: [34]   
     steps:
       - name: Checkout 🛎️
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -114,8 +115,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        api-level: [34]
-        
+        api-level: [34]    
     steps:
       - name: Checkout 🛎️
         uses: actions/checkout@v4
@@ -199,11 +199,13 @@ jobs:
         run:  bash <(curl -s https://codecov.io/bash) -F geowidget -f "geowidget/build/reports/jacoco/fhircoreJacocoReport/fhircoreJacocoReport.xml"
 
   quest-tests:
+    timeout-minutes: 90  # Extend timeout to 90 minutes
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        shard: [0, 1, 2, 3, 4, 5]  # Split into 6 shards
+        num-shards: [6]      # Total number of shards
         api-level: [34]
-        
     steps: 
       - name: Checkout 🛎️
         uses: actions/checkout@v4
@@ -258,7 +260,7 @@ jobs:
           working-directory: android
           api-level: ${{ matrix.api-level }}
           arch: x86_64
-          force-avd-creation: true
+          force-avd-creation: false
           emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: false
           script: echo "Generated AVD snapshot for caching."
@@ -269,45 +271,33 @@ jobs:
           working-directory: android
           api-level: ${{ matrix.api-level }}
           arch: x86_64
-          force-avd-creation: true
-          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
-          disable-animations: true
-          script: ./gradlew clean  -PlocalPropertiesFile=local.properties :quest:fhircoreJacocoReport --stacktrace -Pandroid.testInstrumentationRunnerArguments.notPackage=org.smartregister.fhircore.quest.performance
-
-      - name: Run Quest module unit and instrumentation tests and generate aggregated coverage report (Disabled)
-        if: false
-        uses: reactivecircus/android-emulator-runner@v2
-        with:
-          working-directory: android
-          api-level: ${{ matrix.api-level }}
-          arch: x86_64
-          force-avd-creation: true
+          force-avd-creation: false
+          heap-size: 256M
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true
-          run: |
-            set -e
-            ./gradlew -PlocalPropertiesFile=local.properties :quest:clean
-            ./gradlew -PlocalPropertiesFile=local.properties :quest:assembleOpensrpDebugAndroidTest --stacktrace 
-            /Users/martin/Library/Android/sdk/platform-tools/adb install quest/build/outputs/apk/androidTest/opensrp/debug/quest-opensrp-debug-androidTest.apk
-            ./gradlew -PlocalPropertiesFile=local.properties :quest:assembleOpensrpDebug --stacktrace
-            /Users/martin/Library/Android/sdk/platform-tools/adb install quest/build/outputs/apk/opensrp/debug/quest-opensrp-debug.apk 
-            /Users/martin/Library/Android/sdk/platform-tools/adb shell am instrument -w \
-              --no-window-animation \
-              -e coverage "true" \
-              -e debug false \
-              org.smartregister.opensrp.test/org.smartregister.fhircore.quest.QuestTestRunner
-            /Users/martin/Library/Android/sdk/platform-tools/adb shell run-as org.smartregister.opensrp \
-              cat "/data/user/0/org.smartregister.opensrp/files/coverage.ec" > quest/coverage.ec 
-            ./gradlew -PlocalPropertiesFile=local.properties :quest:fhircoreJacocoReport --stacktrace
+          script: >-
+            ./gradlew -PlocalPropertiesFile=local.properties :quest:fhircoreJacocoReport
+            -Pandroid.testInstrumentationRunnerArguments.notPackage=org.smartregister.fhircore.quest.performance
+            -Pandroid.testInstrumentationRunnerArguments.numShards=${{ matrix.num-shards }}
+            -Pandroid.testInstrumentationRunnerArguments.shardIndex=${{ matrix.shard }}
 
       - name: Upload Test reports
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: quest-test-reports
+          name: quest-test-report-${{ matrix.shard }}
           path: android/quest/build/reports
 
       - name: Upload Quest module test coverage report to Codecov
         if: matrix.api-level == 34 # Only upload coverage on API level 34
         working-directory: android
         run:  bash <(curl -s https://codecov.io/bash) -F quest -f "quest/build/reports/jacoco/fhircoreJacocoReport/fhircoreJacocoReport.xml"
+  
+  quest-success:
+      name: Quest Tests Success
+      needs: quest-tests
+      runs-on: ubuntu-latest
+      if: ${{ !cancelled() }}
+      steps:
+        - name: Quest Tests Passed
+          run: echo "All Quest tests passed!"

android/buildSrc/src/main/kotlin/project-properties.gradle.kts

@@ -27,7 +27,9 @@ val requiredFhirProperties =
     "OAUTH_SCOPE",
     "MAPBOX_SDK_TOKEN",
     "SENTRY_DSN",
-    "OPENSRP_APP_ID"
+    "OPENSRP_APP_ID",
+    "GEMINI_API_KEY",
+    "SPEECH_TO_TEXT_API_KEY",
   )
 
 val localProperties = readProperties((project.properties["localPropertiesFile"] ?: "${rootProject.projectDir}/local.properties").toString())

android/engine/src/main/java/org/smartregister/fhircore/engine/configuration/QuestionnaireConfig.kt

@@ -68,6 +68,7 @@ data class QuestionnaireConfig(
   val linkIds: List<LinkIdConfig>? = null,
   val showSubmitAnywayButton: String = "false",
   val showSubmissionConfirmationDialog: String = "false",
+  val enableSpeechToText: Boolean = false,
 ) : java.io.Serializable, Parcelable {
 
   fun interpolate(computedValuesMap: Map<String, Any>) =

android/engine/src/main/java/org/smartregister/fhircore/engine/util/SharedPreferencesHelper.kt

@@ -24,6 +24,9 @@ import dagger.hilt.android.qualifiers.ApplicationContext
 import javax.inject.Inject
 import javax.inject.Singleton
 import kotlinx.serialization.SerializationException
+import org.jetbrains.annotations.VisibleForTesting
+import org.smartregister.fhircore.engine.BuildConfig
+import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry.Companion.DEBUG_SUFFIX
 import org.smartregister.fhircore.engine.util.extension.decodeJson
 import org.smartregister.fhircore.engine.util.extension.encodeJson
 import timber.log.Timber
@@ -108,6 +111,12 @@ constructor(@ApplicationContext val context: Context, val gson: Gson) {
 
   fun retrieveApplicationId() = read(SharedPreferenceKey.APP_ID.name, null)
 
+  fun hasDebugSuffix(): Boolean =
+    retrieveApplicationId()?.trim()?.endsWith(DEBUG_SUFFIX, ignoreCase = true) == true &&
+      isDebugVariant()
+
+  @VisibleForTesting fun isDebugVariant() = BuildConfig.DEBUG
+
   companion object {
     const val PREFS_NAME = "params"
     const val PREFS_SYNC_PROGRESS_TOTAL = "sync_progress_total"

android/engine/src/test/java/org/smartregister/fhircore/engine/util/SharedPreferencesHelperTest.kt

@@ -22,6 +22,8 @@ import androidx.test.core.app.ApplicationProvider
 import com.google.gson.Gson
 import dagger.hilt.android.testing.HiltAndroidRule
 import dagger.hilt.android.testing.HiltAndroidTest
+import io.mockk.coEvery
+import io.mockk.spyk
 import javax.inject.Inject
 import org.junit.Assert
 import org.junit.Before
@@ -122,4 +124,38 @@ internal class SharedPreferencesHelperTest : RobolectricTest() {
 
     Assert.assertEquals("test.demo", retrievedAppId)
   }
+
+  @Test
+  fun testHasDebugSuffix_withSuffix_shouldReturn_true() {
+    setDebugInvariantTrue()
+    sharedPreferencesHelper.write(SharedPreferenceKey.APP_ID.name, "app/debug")
+    Assert.assertTrue(sharedPreferencesHelper.hasDebugSuffix())
+  }
+
+  @Test
+  fun testHasDebugSuffix_noSuffix_shouldReturn_false() {
+    setDebugInvariantTrue()
+    sharedPreferencesHelper.write(SharedPreferenceKey.APP_ID.name, "app")
+    Assert.assertFalse(sharedPreferencesHelper.hasDebugSuffix())
+  }
+
+  @Test
+  fun testHasDebugSuffix_emptyAppId_shouldReturn_null() {
+    setDebugInvariantTrue()
+    sharedPreferencesHelper.write(SharedPreferenceKey.APP_ID.name, null)
+    Assert.assertFalse(sharedPreferencesHelper.hasDebugSuffix())
+  }
+
+  @Test
+  fun testIsDebugVariant_shouldReturnTrue() {
+    Assert.assertTrue(sharedPreferencesHelper.isDebugVariant())
+  }
+
+  private fun setDebugInvariantTrue() {
+    sharedPreferencesHelper =
+      spyk(
+        SharedPreferencesHelper(application, gson),
+      )
+    coEvery { sharedPreferencesHelper.isDebugVariant() } returns true
+  }
 }

android/geowidget/src/main/java/org/smartregister/fhircore/geowidget/screens/GeoWidgetViewModel.kt

@@ -26,6 +26,47 @@ class GeoWidgetViewModel : ViewModel() {
 
   val mapFeatures = ArrayDeque<Feature>()
 
+  private val servicePointTypes =
+    listOf(
+      ServicePointType.EPP,
+      ServicePointType.CEG,
+      ServicePointType.CHRD1,
+      ServicePointType.CHRD2,
+      ServicePointType.DRSP,
+      ServicePointType.MSP,
+      ServicePointType.SDSP,
+      ServicePointType.CSB1,
+      ServicePointType.CSB2,
+      ServicePointType.CHRR,
+      ServicePointType.WAREHOUSE,
+      ServicePointType.WATER_POINT,
+      ServicePointType.PRESCO,
+      ServicePointType.MEAH,
+      ServicePointType.DREAH,
+      ServicePointType.MPPSPF,
+      ServicePointType.DRPPSPF,
+      ServicePointType.NGO_PARTNER,
+      ServicePointType.SITE_COMMUNAUTAIRE,
+      ServicePointType.DRJS,
+      ServicePointType.INSTAT,
+      ServicePointType.BSD,
+      ServicePointType.MEN,
+      ServicePointType.DREN,
+      ServicePointType.DISTRICT_PPSPF,
+      ServicePointType.MAIRIE,
+      ServicePointType.ECOLE_COMMUNAUTAIRE,
+      ServicePointType.ECOLE_PRIVÉ,
+      ServicePointType.ECOLE_PUBLIQUE,
+      ServicePointType.CENTRE_DE_SANTE,
+      ServicePointType.CENTRE_DE_TRAITEMENT_DU_CHOLERA,
+      ServicePointType.HOPITAL_COMMUNAL,
+      ServicePointType.HOPITAL,
+      ServicePointType.BUREAU_DES_PARTENAIRES,
+      ServicePointType.LYCÉE,
+      ServicePointType.DIRECTION_COMMUNALE_DE_L_ENSEIGNEMENT,
+      ServicePointType.ECOLE_PRIVE,
+    )
+
   fun updateMapFeatures(geoJsonFeatures: List<GeoJsonFeature>) {
     if (mapFeatures.size <= MAP_FEATURES_LIMIT) {
       mapFeatures.addAll(geoJsonFeatures.map { it.toFeature() })
@@ -34,50 +75,8 @@ class GeoWidgetViewModel : ViewModel() {
 
   fun clearMapFeatures() = mapFeatures.clear()
 
-  // Todo Refactor this to work different and better
   fun getServicePointKeyToType(): Map<String, ServicePointType> {
-    val map: MutableMap<String, ServicePointType> = HashMap()
-    map[ServicePointType.EPP.name.lowercase()] = ServicePointType.EPP
-    map[ServicePointType.CEG.name.lowercase()] = ServicePointType.CEG
-    map[ServicePointType.CHRD1.name.lowercase()] = ServicePointType.CHRD1
-    map[ServicePointType.CHRD2.name.lowercase()] = ServicePointType.CHRD2
-    map[ServicePointType.DRSP.name.lowercase()] = ServicePointType.DRSP
-    map[ServicePointType.MSP.name.lowercase()] = ServicePointType.MSP
-    map[ServicePointType.SDSP.name.lowercase()] = ServicePointType.SDSP
-    map[ServicePointType.CSB1.name.lowercase()] = ServicePointType.CSB1
-    map[ServicePointType.CSB2.name.lowercase()] = ServicePointType.CSB2
-    map[ServicePointType.CHRR.name.lowercase()] = ServicePointType.CHRR
-    map[ServicePointType.WAREHOUSE.name.lowercase()] = ServicePointType.WAREHOUSE
-    map[ServicePointType.WATER_POINT.name.lowercase()] = ServicePointType.WATER_POINT
-    map[ServicePointType.PRESCO.name.lowercase()] = ServicePointType.PRESCO
-    map[ServicePointType.MEAH.name.lowercase()] = ServicePointType.MEAH
-    map[ServicePointType.DREAH.name.lowercase()] = ServicePointType.DREAH
-    map[ServicePointType.MPPSPF.name.lowercase()] = ServicePointType.MPPSPF
-    map[ServicePointType.DRPPSPF.name.lowercase()] = ServicePointType.DRPPSPF
-    map[ServicePointType.NGO_PARTNER.name.lowercase()] = ServicePointType.NGO_PARTNER
-    map[ServicePointType.SITE_COMMUNAUTAIRE.name.lowercase()] = ServicePointType.SITE_COMMUNAUTAIRE
-    map[ServicePointType.DRJS.name.lowercase()] = ServicePointType.DRJS
-    map[ServicePointType.INSTAT.name.lowercase()] = ServicePointType.INSTAT
-    map[ServicePointType.BSD.name.lowercase()] = ServicePointType.BSD
-    map[ServicePointType.MEN.name.lowercase()] = ServicePointType.MEN
-    map[ServicePointType.DREN.name.lowercase()] = ServicePointType.DREN
-    map[ServicePointType.DISTRICT_PPSPF.name.lowercase()] = ServicePointType.DISTRICT_PPSPF
-    map[ServicePointType.MAIRIE.name.lowercase()] = ServicePointType.MAIRIE
-    map[ServicePointType.ECOLE_COMMUNAUTAIRE.name.lowercase()] =
-      ServicePointType.ECOLE_COMMUNAUTAIRE
-    map[ServicePointType.ECOLE_PRIVÉ.name.lowercase()] = ServicePointType.ECOLE_PRIVÉ
-    map[ServicePointType.ECOLE_PUBLIQUE.name.lowercase()] = ServicePointType.ECOLE_PUBLIQUE
-    map[ServicePointType.CENTRE_DE_SANTE.name.lowercase()] = ServicePointType.CENTRE_DE_SANTE
-    map[ServicePointType.CENTRE_DE_TRAITEMENT_DU_CHOLERA.name.lowercase()] =
-      ServicePointType.CENTRE_DE_TRAITEMENT_DU_CHOLERA
-    map[ServicePointType.HOPITAL_COMMUNAL.name.lowercase()] = ServicePointType.HOPITAL_COMMUNAL
-    map[ServicePointType.HOPITAL.name.lowercase()] = ServicePointType.HOPITAL
-    map[ServicePointType.BUREAU_DES_PARTENAIRES.name.lowercase()] =
-      ServicePointType.BUREAU_DES_PARTENAIRES
-    map[ServicePointType.LYCÉE.name.lowercase()] = ServicePointType.LYCÉE
-    map[ServicePointType.DIRECTION_COMMUNALE_DE_L_ENSEIGNEMENT.name.lowercase()] =
-      ServicePointType.DIRECTION_COMMUNALE_DE_L_ENSEIGNEMENT
-    map[ServicePointType.ECOLE_PRIVE.name.lowercase()] = ServicePointType.ECOLE_PRIVE
-    return map
+    // Use the class-level list to create the map
+    return servicePointTypes.associateBy { it.name.lowercase() }
   }
 }

android/geowidget/src/test/java/org/smartregister/fhircore/geowidget/screens/GeoWidgetViewModelTest.kt

@@ -113,39 +113,12 @@ class GeoWidgetViewModelTest {
 
   @Test
   fun testThatMapOfServicePointTypeReturnsEnumValuesBasedOnTheirLowercaseNames() {
-    val result = geoWidgetViewModel.getServicePointKeyToType()
-    val expectedMap =
-      mapOf(
-        "epp" to ServicePointType.EPP,
-        "ceg" to ServicePointType.CEG,
-        "chrd1" to ServicePointType.CHRD1,
-        "chrd2" to ServicePointType.CHRD2,
-        "drsp" to ServicePointType.DRSP,
-        "msp" to ServicePointType.MSP,
-        "sdsp" to ServicePointType.SDSP,
-        "csb1" to ServicePointType.CSB1,
-        "csb2" to ServicePointType.CSB2,
-        "chrr" to ServicePointType.CHRR,
-        "warehouse" to ServicePointType.WAREHOUSE,
-        "water_point" to ServicePointType.WATER_POINT,
-        "presco" to ServicePointType.PRESCO,
-        "meah" to ServicePointType.MEAH,
-        "dreah" to ServicePointType.DREAH,
-        "mppspf" to ServicePointType.MPPSPF,
-        "drppspf" to ServicePointType.DRPPSPF,
-        "ngo_partner" to ServicePointType.NGO_PARTNER,
-        "site_communautaire" to ServicePointType.SITE_COMMUNAUTAIRE,
-        "drjs" to ServicePointType.DRJS,
-        "instat" to ServicePointType.INSTAT,
-        "bsd" to ServicePointType.BSD,
-        "men" to ServicePointType.MEN,
-        "dren" to ServicePointType.DREN,
-        "district_ppspf" to ServicePointType.DISTRICT_PPSPF,
-        "mairie" to ServicePointType.MAIRIE,
-        "ecole_communautaire" to ServicePointType.ECOLE_COMMUNAUTAIRE,
-        "ecole_privé" to ServicePointType.ECOLE_PRIVÉ,
-        "lycée" to ServicePointType.LYCÉE,
-      )
-    Assert.assertEquals(expectedMap, result)
+    val servicePointTypeMap = geoWidgetViewModel.getServicePointKeyToType()
+
+    ServicePointType.entries.forEach { servicePointType ->
+      val expectedValue = servicePointType.name.lowercase()
+      val actualValue = servicePointTypeMap[expectedValue]
+      Assert.assertEquals(servicePointType, actualValue)
+    }
   }
 }

android/gradle.properties

@@ -20,11 +20,13 @@ android.useAndroidX=true
 android.enableJetifier=true
 android.defaults.buildfeatures.buildconfig=true
 android.suppressUnsupportedCompileSdk=34
+android.testInstrumentationRunnerArguments.useTestStorageService=true
 android.jetifier.ignorelist=jackson-core
 #org.gradle.daemon.max-idle-time=10800000  // 3 hours in milliseconds
 
 org.gradle.warning.mode=all
-org.gradle.configuration-cache=true
+# set configuration-cache to false in order to avoid macrobenchmark failure
+org.gradle.configuration-cache=false
 org.gradle.caching=false
 org.gradle.parallel=true
 org.gradle.configureondemand=true