Skip to content

Commit 6b7b30c

Browse files
LZRSLZRS
authored
Compose benchmark app crud (#2818)
* Hide synthea directory * Add CRUD trace and elapsed-time benchmarks * Update CRUD Ui view to show average time * Add testTags for interop with uiautomator in macrobenchmarks https://developer.android.com/develop/ui/compose/testing/interoperability#uiautomator-interop * Resolve PR comments on BenchmarkResult and BenchmarkDuration * Use suspending functions to avoid runBlocking in CrudApiViewmodel * Fix crash when reading from multiple .ndjson files * Set configurable sample size of 500 for read/update/delete * Add Macrobenchmarks module with ftl testing * Add macrobenchmark tests module for engine benchmarks * Update FhirEngineCrudBenchmark tracing to use testTags * Update FhirEngineCrudBenchmark to wait until benchmark completes * Update runFlank script with new engine benchmarks dir structure * Register generateSynthea task to run Synthea through gradle * Add ftl testing for the macrobenchmark module * Update generate synthea to copy all .ndjson files to assets * Update reading of resources data from assets * Use a population of 100k * Add documentation for FhirEngine benchmarking with CRUD api * Reduce synthea population in kokoro to avoid crashing * Update docs with steps on accessing benchmark results from Kokoro * Merge the separate benchmark sections into single page in doc
1 parent a799f41 commit 6b7b30c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+676
-142
lines changed

‎.github/workflows/runFlank.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# Fail on any error.
1919
set -e
2020

21-
lib_names=("workflow:benchmark" "engine:benchmark" "datacapture" "engine" "knowledge" "workflow")
21+
lib_names=("workflow:benchmark" "engine:benchmarks:macrobenchmark" "engine:benchmarks:microbenchmark" "datacapture" "engine" "knowledge" "workflow")
2222
firebase_pids=()
2323

2424
for lib_name in "${lib_names[@]}"; do

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,6 @@ docs/use/api/*/**
9797

9898
# Kotlin 2.0
9999
.kotlin/
100+
101+
# Synthea
102+
synthea

‎build.gradle.kts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,8 @@ allprojects {
2929
}
3030

3131
subprojects {
32-
// We have some empty folders like the :contrib root folder, which Gradle recognizes as projects.
33-
// Don't configure plugins for those folders.
34-
if (project.buildFile.exists()) {
35-
configureLicensee()
36-
}
32+
applyLicenseeConfig()
33+
3734
tasks.withType(Test::class.java).configureEach {
3835
maxParallelForks = 1
3936
if (project.providers.environmentVariable("GITHUB_ACTIONS").isPresent) {

‎buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt‎

Lines changed: 43 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ fun Project.configureFirebaseTestLabForLibraries() {
2525
apply(plugin = Plugins.BuildPlugins.fladle)
2626
configure<FlankGradleExtension> {
2727
commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLabForLibraries)
28+
debugApk.set(
29+
project.provider {
30+
"${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
31+
},
32+
)
2833
instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/debug/*.apk" })
2934
environmentVariables.set(
3035
mapOf(
@@ -51,37 +56,54 @@ fun Project.configureFirebaseTestLabForLibraries() {
5156
}
5257
}
5358

59+
fun Project.configureFirebaseTestLabForMacroBenchmark() {
60+
apply(plugin = Plugins.BuildPlugins.fladle)
61+
configure<FlankGradleExtension> {
62+
commonConfigurationForFirebaseTestLabBenchmark(this@configureFirebaseTestLabForMacroBenchmark)
63+
debugApk.set(
64+
project.provider {
65+
"${project.rootDir}/engine/benchmarks/app/build/outputs/apk/benchmark/app-benchmark.apk"
66+
},
67+
)
68+
instrumentationApk.set(project.provider { "$buildDir/outputs/apk/benchmark/*.apk" })
69+
}
70+
}
71+
5472
fun Project.configureFirebaseTestLabForMicroBenchmark() {
5573
apply(plugin = Plugins.BuildPlugins.fladle)
5674
configure<FlankGradleExtension> {
57-
commonConfigurationForFirebaseTestLab(this@configureFirebaseTestLabForMicroBenchmark)
75+
commonConfigurationForFirebaseTestLabBenchmark(this@configureFirebaseTestLabForMicroBenchmark)
76+
debugApk.set(
77+
project.provider {
78+
"${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
79+
},
80+
)
5881
instrumentationApk.set(project.provider { "$buildDir/outputs/apk/androidTest/release/*.apk" })
59-
environmentVariables.set(
82+
}
83+
}
84+
85+
private fun FlankGradleExtension.commonConfigurationForFirebaseTestLabBenchmark(project: Project) {
86+
commonConfigurationForFirebaseTestLab(project)
87+
environmentVariables.set(
88+
mapOf(
89+
"additionalTestOutputDir" to "/sdcard/Download",
90+
"no-isolated-storage" to "true",
91+
"clearPackageData" to "true",
92+
),
93+
)
94+
devices.set(
95+
listOf(
6096
mapOf(
61-
"additionalTestOutputDir" to "/sdcard/Download",
62-
"no-isolated-storage" to "true",
63-
"clearPackageData" to "true",
97+
"model" to "panther",
98+
"version" to "33",
99+
"locale" to "en_US",
64100
),
65-
)
66-
devices.set(
67-
listOf(
68-
mapOf(
69-
"model" to "panther",
70-
"version" to "33",
71-
"locale" to "en_US",
72-
),
73-
),
74-
)
75-
}
101+
),
102+
)
76103
}
77104

78105
private fun FlankGradleExtension.commonConfigurationForFirebaseTestLab(project: Project) {
79106
projectId.set("android-fhir-instrumeted-tests")
80-
debugApk.set(
81-
project.provider {
82-
"${project.rootDir}/demo/build/outputs/apk/androidTest/debug/demo-debug-androidTest.apk"
83-
},
84-
)
85107
useOrchestrator.set(true)
86108
flakyTestAttempts.set(1)
87109
maxTestShards.set(10)

‎buildSrc/src/main/kotlin/LicenseeConfig.kt‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2023-2024 Google LLC
2+
* Copyright 2023-2025 Google LLC
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -18,7 +18,23 @@ import org.gradle.api.Project
1818
import org.gradle.kotlin.dsl.apply
1919
import org.gradle.kotlin.dsl.configure
2020

21-
fun Project.configureLicensee() {
21+
fun Project.applyLicenseeConfig() {
22+
// Skip project ":engine:benchmarks:macrobenchmark" since it's a "com.android.test" project
23+
// which is not compatible with Licensee
24+
if (project.path == ":engine:benchmarks:macrobenchmark") {
25+
return
26+
}
27+
28+
// We have some empty folders like the :contrib root folder, which Gradle recognizes as projects.
29+
// Don't configure plugins for those folders.
30+
if (!project.buildFile.exists()) {
31+
return
32+
}
33+
34+
configureLicensee()
35+
}
36+
37+
private fun Project.configureLicensee() {
2238
apply(plugin = "app.cash.licensee")
2339
configure<app.cash.licensee.LicenseeExtension> {
2440
allow("Apache-2.0")

‎buildSrc/src/main/kotlin/Plugins.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ object Plugins {
3131
const val navSafeArgs = "androidx.navigation.safeargs.kotlin"
3232
const val ruler = "com.spotify.ruler"
3333
const val spotless = "com.diffplug.spotless"
34+
const val androidTest = "com.android.test"
3435
}
3536

3637
// classpath plugins
@@ -50,7 +51,7 @@ object Plugins {
5051

5152
object Versions {
5253
const val androidGradlePlugin = "8.9.2"
53-
const val benchmarkPlugin = "1.1.0"
54+
const val benchmarkPlugin = "1.3.4"
5455
const val dokka = "1.9.20"
5556
const val kspPlugin = "2.1.20-2.0.1"
5657
const val kotlin = "2.1.20"

‎buildSrc/src/main/kotlin/SyntheaTaskConfig.kt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import org.gradle.api.Project
18+
import org.gradle.api.tasks.Exec
19+
20+
fun Project.configureSyntheaTask() {
21+
tasks.register("generateSynthea", Exec::class.java) {
22+
val assetsDirPath = "${projectDir.path}/src/main/assets/bulk_data"
23+
val populationSize = providers.gradleProperty("population")
24+
val scriptPath = "${rootDir.path}/generate_synthea.sh"
25+
26+
doFirst {
27+
val scriptArgs = arrayOf("sh", scriptPath, populationSize.orNull ?: "50", assetsDirPath)
28+
commandLine(*scriptArgs)
29+
}
30+
}
31+
}

‎docs/use/FEL/Benchmarking.md‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
# _FHIR Engine Library_ Benchmarks
2+
3+
Benchmarks have been added in the _FHIR Engine Library_ to help track performance regressions
4+
and areas for performance improvement.
5+
6+
The _FHIR Engine Library_ has the following benchmark modules
7+
8+
1. **app** - configurable android application for testing and running benchmarks
9+
2. **microbenchmark** - [Jetpack Microbenchmark](https://developer.android.com/topic/performance/benchmarking/microbenchmark-overview) module
10+
3. **macrobenchmark** - [Jetpack Macrobenchmark](https://developer.android.com/topic/performance/benchmarking/macrobenchmark-overview) module
11+
to automate testing large-scale user-facing apis with the benchmark app
12+
13+
## App module
14+
15+
Located in the module `:engine:benchmarks:app`
16+
The _FHIR Engine Library_ Benchmark app runs benchmarks for the _FHIR Engine Library_ APIs including Data Access API, Search API and Sync API.
17+
18+
It can be configured to run the benchmarks for different population sizes whereby population refers to the number of Patients and their associated data
19+
20+
### Configuration
21+
22+
The benchmark app requires that the dataset that is to be benchmarked on be added in the _assets/bulk_data_ folder as `.ndjson` formatted files; whereby each line refers to a single FHIR resource
23+
24+
![Path to bulk data](path-bulk-data.png)
25+
26+
The dataset can be from an external source. Within the repository, there is a script to generate [synthetic data](https://github.com/synthetichealth/synthea/wiki/Getting-Started) that could then be used for benchmarking.
27+
28+
```shell
29+
./gradlew :engine:benchmarks:app:generateSynthea -Ppopulation=1000
30+
```
31+
32+
It generates [synthetic data](https://github.com/synthetichealth/synthea/wiki/Getting-Started) with a population size of 1000.
33+
The `population` parameter determines the population size that would be used to generate the data
34+
35+
### Running
36+
37+
To run this app in Android Studio, [create a run/debug configuration](https://developer.android.com/studio/run/rundebugconfig) for the `:engine:benchmarks:app` module using the [Android App](https://developer.android.com/studio/run/rundebugconfig#android-application) template and run the app using the configuration.
38+
39+
[Change the build variant](https://developer.android.com/studio/run#changing-variant) to `benchmark` for an optimised version of the app, for the best results
40+
41+
Alternatively, run the following command to build and install the benchmark APK on your device/emulator:
42+
43+
```shell
44+
./gradlew :engine:benchmarks:app:installBenchmark
45+
```
46+
47+
## Microbenchmark module
48+
49+
Contains test cases that evaluate the performance of individual tasks executed for the first time directly on hardware, located in the module `:engine:benchmarks:microbenchmark`.
50+
51+
The test cases are designed to run in sequence of their alphabetic order to make sure larger tasks do not build cache for smaller ones. Their class names are prefixed by an extra letter to inform their position relative to others in the list.
52+
53+
### Running
54+
55+
In Android Studio, set your build variants to `release` and run your benchmark as you would any `@Test` using the gutter action next to your test class or method.
56+
57+
![gutter test action](https://developer.android.com/static/topic/performance/images/benchmark_images/microbenchmark_run.png)
58+
59+
The results will be similar to this:
60+
```
61+
1,297,374 ns 5345 allocs trace EngineDatabaseBenchmark.createAndGet
62+
1,114,474,793 ns 4922289 allocs trace FhirSyncWorkerBenchmark.oneTimeSync_50patients
63+
15,251,125 ns 100542 allocs trace FhirSyncWorkerBenchmark.oneTimeSync_1patient
64+
179,806,709 ns 986017 allocs trace FhirSyncWorkerBenchmark.oneTimeSync_10patients
65+
1,451,758 ns 11883 allocs trace GzipUploadInterceptorBenchmark.upload_10patientsWithGzip
66+
1,537,559 ns 11829 allocs trace GzipUploadInterceptorBenchmark.upload_10patientsWithoutGzip
67+
73,640,833 ns 1074360 allocs trace GzipUploadInterceptorBenchmark.upload_1000patientsWithGzip
68+
7,493,642 ns 108428 allocs trace GzipUploadInterceptorBenchmark.upload_100patientsWithoutGzip
69+
7,799,264 ns 108465 allocs trace GzipUploadInterceptorBenchmark.upload_100patientsWithGzip
70+
71,189,333 ns 1074466 allocs trace GzipUploadInterceptorBenchmark.upload_1000patientsWithoutGzip
71+
72+
```
73+
74+
Alternatively, from the command line, run the connectedCheck to run all of the tests from specified Gradle module:
75+
76+
```bash
77+
./gradlew :engine:benchmarks:microbenchmark:connectedReleaseAndroidTest
78+
```
79+
80+
In this case, results will be saved to the `outputs/androidTest-results/connected/<device>/test-result.pb`. To visualize on Android Studio, click Run / Import Tests From File and find the `.pb` file
81+
82+
### Continuous Integration (CI)
83+
84+
#### Configuration
85+
86+
Microbenchmark tests are configured to run in Kokoro and use [Fladle](https://runningcode.github.io/fladle/) plugin, configured through `Project.configureFirebaseTestLabForMicroBenchmark` in file `buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt`
87+
88+
#### Accessing the benchmark results
89+
90+
The Microbenchmark results can be accessed through the following steps
91+
92+
1. Click to `View details` of the `Kokoro: Build and Device Tests`
93+
![PR Kokoro view details](pr-kokoro-view-details.png)
94+
95+
The details page would look similar to
96+
![Kokoro details](kokoro-details-page.png)
97+
2. Within the `Target Log` tab, locate for the section
98+
![Microbenchmark section](microbenchmark-section.png)
99+
with the `TEST FILE NAME` `microbenchmark-release-androidTest.apk`
100+
3. Select and visit the Google Bucket url that looks as similar to
101+
[https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5404/20250618-172425/firebase/microbenchmark](ttps://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5404/20250618-172425/firebase/microbenchmark)
102+
that navigates to the `android-fhir-build-artifacts` ![bucket](microbenchmark-bucket.png)
103+
4. Navigate to `matrix_0/panther-33-en_US-portrait-test_results_merged.xml` to download the benchmark .xml results file. The `panther-33-en_US-portrait` in the path refers to the Firebase Test Lab device used in running the benchmark tests.
104+
105+
## Macrobenchmark module
106+
107+
The _FHIR Engine Library_ macrobenchmark tests are located in the module `:engine:benchmarks:macrobenchmark`
108+
109+
### Prerequisite
110+
111+
Requires the _FHIR Engine Library_ Benchmark App configured with the relevant benchmark data described in the section for the _FHIR Engine Library_ Benchmark App
112+
113+
### Running
114+
115+
To run, use the command
116+
117+
```shell
118+
./gradlew :engine:benchmarks:macrobenchmark:connectedCheck
119+
```
120+
121+
### Continuous Integration (CI)
122+
123+
#### Configuration
124+
125+
The `FHIR Engine` Macrobenchmarks have been configured to run in Kokoro and use FirebaseTestLab physical devices
126+
127+
Configuration for the Kokoro script are currently located in `kokoro/gcp_ubuntu/kokoro_build.sh` while the FirebaseTestLab testing is configured through the [Fladle](https://runningcode.github.io/fladle/) plugin in `buildSrc/src/main/kotlin/FirebaseTestLabConfig.kt`
128+
129+
#### Accessing the benchmark results
130+
131+
From a GitHub PR , the following steps could be used to download the benchmark results from a Kokoro run
132+
133+
1. Click to `View details` of the `Kokoro: Build and Device Tests`
134+
![PR Kokoro view details](pr-kokoro-view-details.png)
135+
136+
The details page would look similar to
137+
![Kokoro details](kokoro-details-page.png)
138+
139+
2. Within the `Target Log` tab, locate for the section
140+
![Macrobenchmark section](macrobenchmark-section.png)
141+
with the `TEST FILE NAME` `macrobenchmark-benchmark.apk`
142+
143+
3. Select and visit the url as referenced in image
144+
![Reference url](select-bucket-url.png)
145+
representative of the Google Cloud Bucket containing the artifacts from the Kokoro run. From the image example, the url is [https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5403/20250616-053647/firebase/macrobenchmark](https://console.developers.google.com/storage/browser/android-fhir-build-artifacts/prod/openhealthstack/android-fhir/gcp_ubuntu/presubmit/5403/20250616-053647/firebase/macrobenchmark)
146+
147+
The bucket page would look similar to
148+
![Artifacts bucket page](google-bucket-page.png)
149+
150+
4. Navigate to `matrix_0/panther-33-en_US-portrait/artifacts/sdcard/Download/com.google.android.fhir.engine.macrobenchmark-benchmarkData.json` to download the benchmark results file. The `panther-33-en_US-portrait` in the path represents the Firebase Test Lab device that was used to run the benchmark tests.
151+
152+
#### Sample Benchmark Results
153+
154+
The results shared below are generated from running the _FHIR Engine Library_ Macrobenchmark tests in Kokoro
155+
156+
##### [**_Panther - Google Pixel 7_**](https://wiki.lineageos.org/devices/panther/)
157+
158+
**CPU** - Octa-core (2x2.85 GHz Cortex-X1 & 2x2.35 GHz Cortex-A78 & 4x1.80 GHz Cortex-A55)
159+
160+
**RAM** - 8GB
161+
162+
API 33
163+
164+
###### Data Access API results
165+
166+
Results were generated from execution of FhirEngineCrudBenchmark test in the `engine:benchmarks:macrobenchmark` module located at `engine/benchmarks/macrobenchmark/src/main/java/com/google/android/fhir/engine/macrobenchmark/FhirEngineCrudBenchmark.kt`
167+
168+
| API | Average duration (ms) | Notes |
169+
|:-------|----------------------:|---------------------------------------|
170+
| create | ~4.7 | Takes ~47s for population size of 10k |
171+
| update | ~12.29 | |
172+
| get | ~3.83 | |
173+
| delete | ~8.08 | |

‎docs/use/FEL/google-bucket-page.png‎

544 KB
Loading

‎docs/use/FEL/kokoro-details-page.png‎

506 KB
Loading

0 commit comments

Comments
 (0)