Add automated baseline profile generation (#880)

Baseline profile generation is disabled for the PR level Build task. Release tasks require a fresh baseline profile. A new profile is generated using the baseline profile Gradle plugin.

* Prepare for usage of dex layout optimizations which can be actively used once NiA switches to AGP 8.2+.
* Add GMD config to release build
* Switch to macos-latest
* Update names for StartupBenchmark tests to better reflect states
* Stable release and recent GMD device
* Reduce flakiness by adding wait to benchmark
* More convenient waiting for objects
* Rename junit dependency to androidx-junit
* Only run baseline profile benchmarks during GH workflow
* Enable automatic BP generation for only release builds
* Disable BP generation from Build workflow
* Specify modules and skip benchmarking Build workflow

Bug: b/299334172

Author: keyboardsurfer

.github/workflows/Build.yaml

@@ -73,9 +73,19 @@ jobs:
       - name: Run local tests
         if: always()
         run: ./gradlew testDemoDebug testProdDebug :lint:test
-
+      # Replace task exclusions with `-Pandroidx.baselineprofile.skipgeneration` when
+      # https://android-review.googlesource.com/c/platform/frameworks/support/+/2602790 landed in a
+      # release build
       - name: Build all build type and flavor permutations
-        run: ./gradlew assemble
+        run: ./gradlew :app:assemble :benchmarks:assemble
+          -x pixel6Api33ProdNonMinifiedReleaseAndroidTest
+          -x pixel6Api33ProdNonMinifiedBenchmarkAndroidTest
+          -x pixel6Api33DemoNonMinifiedReleaseAndroidTest
+          -x pixel6Api33DemoNonMinifiedBenchmarkAndroidTest
+          -x collectDemoNonMinifiedReleaseBaselineProfile
+          -x collectDemoNonMinifiedBenchmarkBaselineProfile
+          -x collectProdNonMinifiedReleaseBaselineProfile
+          -x collectProdNonMinifiedBenchmarkBaselineProfile
 
       - name: Upload build outputs (APKs)
         uses: actions/upload-artifact@v3

.github/workflows/Release.yml

@@ -7,8 +7,8 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
+    runs-on: macos-latest
+    timeout-minutes: 120
 
     steps:
       - name: Checkout
@@ -26,9 +26,19 @@ jobs:
           distribution: 'zulu'
           java-version: 17
 
-      - name: Build app
-        run: ./gradlew :app:assembleDemoRelease
+      - name: Install GMD image for baseline profile generation
+        run: yes | "$ANDROID_HOME"/cmdline-tools/latest/bin/sdkmanager "system-images;android-33;aosp_atd;x86_64"
+
+      - name: Accept Android licenses
+        run: yes | "$ANDROID_HOME"/cmdline-tools/latest/bin/sdkmanager --licenses || true
 
+      - name: Build release variant including baseline profile generation
+        run: ./gradlew :app:assembleDemoRelease
+             -Pandroid.testInstrumentationRunnerArguments.androidx.benchmark.enabledRules=BaselineProfile
+             -Pandroid.testoptions.manageddevices.emulator.gpu="swiftshader_indirect"
+             -Pandroid.experimental.testOptions.managedDevices.emulator.showKernelLogging=true
+             -Pandroid.experimental.androidTest.numManagedDeviceShards=1
+             -Pandroid.experimental.testOptions.managedDevices.maxConcurrentDevices=1
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1

app/build.gradle.kts

@@ -24,6 +24,7 @@ plugins {
     id("jacoco")
     alias(libs.plugins.nowinandroid.android.application.firebase)
     id("com.google.android.gms.oss-licenses-plugin")
+    alias(libs.plugins.baselineprofile)
 }
 
 android {
@@ -43,7 +44,7 @@ android {
         debug {
             applicationIdSuffix = NiaBuildType.DEBUG.applicationIdSuffix
         }
-        val release by getting {
+        val release = getByName("release") {
             isMinifyEnabled = true
             applicationIdSuffix = NiaBuildType.RELEASE.applicationIdSuffix
             proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
@@ -52,6 +53,8 @@ android {
             // who clones the code to sign and run the release variant, use the debug signing key.
             // TODO: Abstract the signing configuration to a separate file to avoid hardcoding this.
             signingConfig = signingConfigs.getByName("debug")
+            // Ensure Baseline Profile is fresh for release builds.
+            baselineProfile.automaticGenerationDuringBuild = true
         }
         create("benchmark") {
             // Enable all the optimizations from release build through initWith(release).
@@ -121,6 +124,8 @@ dependencies {
     implementation(libs.kotlinx.coroutines.guava)
     implementation(libs.coil.kt)
 
+    baselineProfile(project(":benchmarks"))
+
     // Core functions
     testImplementation(projects.core.testing)
     testImplementation(projects.core.datastoreTest)
@@ -133,3 +138,9 @@ dependencies {
     kspTest(libs.hilt.compiler)
 
 }
+
+baselineProfile {
+    // Don't build on every iteration of a full assemble.
+    // Instead enable generation directly for the release build variant.
+    automaticGenerationDuringBuild = false
+}

benchmarks/build.gradle.kts

@@ -17,6 +17,7 @@ import com.google.samples.apps.nowinandroid.NiaBuildType
 import com.google.samples.apps.nowinandroid.configureFlavors
 
 plugins {
+    alias(libs.plugins.baselineprofile)
     alias(libs.plugins.nowinandroid.android.test)
 }
 
@@ -62,10 +63,27 @@ android {
         )
     }
 
+    testOptions.managedDevices.devices {
+        create<com.android.build.api.dsl.ManagedVirtualDevice>("pixel6Api33") {
+            device = "Pixel 6"
+            apiLevel = 33
+            systemImageSource = "aosp"
+        }
+    }
+
     targetProjectPath = ":app"
     experimentalProperties["android.experimental.self-instrumenting"] = true
 }
 
+baselineProfile {
+    // This specifies the managed devices to use that you run the tests on.
+    managedDevices += "pixel6Api33"
+
+    // Don't use a connected device but rely on a GMD for consistency between local and CI builds.
+    useConnectedDevices = false
+
+}
+
 dependencies {
     implementation(libs.androidx.benchmark.macro)
     implementation(libs.androidx.test.core)
@@ -75,9 +93,3 @@ dependencies {
     implementation(libs.androidx.test.runner)
     implementation(libs.androidx.test.uiautomator)
 }
-
-androidComponents {
-    beforeVariants {
-        it.enable = it.buildType == "benchmark"
-    }
-}

benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/GeneralActions.kt

@@ -20,6 +20,10 @@ import android.Manifest.permission
 import android.os.Build.VERSION.SDK_INT
 import android.os.Build.VERSION_CODES.TIRAMISU
 import androidx.benchmark.macro.MacrobenchmarkScope
+import androidx.test.uiautomator.By
+import androidx.test.uiautomator.BySelector
+import androidx.test.uiautomator.UiObject2
+import androidx.test.uiautomator.Until
 
 /**
  * Because the app under test is different from the one running the instrumentation test,
@@ -42,3 +46,27 @@ fun MacrobenchmarkScope.allowNotifications() {
         device.executeShellCommand(command)
     }
 }
+
+/**
+ * Wraps starting the default activity, waiting for it to start and then allowing notifications in
+ * one convenient call.
+ */
+fun MacrobenchmarkScope.startActivityAndAllowNotifications() {
+    startActivityAndWait()
+    allowNotifications()
+}
+
+/**
+ * Waits for and returns the `niaTopAppBar`
+ */
+fun MacrobenchmarkScope.getTopAppBar(): UiObject2 {
+    device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
+    return device.findObject(By.res("niaTopAppBar"))
+}
+
+/**
+ * Waits for an object on the top app bar, passed in as [selector].
+ */
+fun MacrobenchmarkScope.waitForObjectOnTopAppBar(selector: BySelector, timeout: Long = 2_000) {
+    getTopAppBar().wait(Until.hasObject(selector), timeout)
+}

benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/BookmarksBaselineProfile.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.samples.apps.nowinandroid.baselineprofile
+
+import androidx.benchmark.macro.junit4.BaselineProfileRule
+import com.google.samples.apps.nowinandroid.PACKAGE_NAME
+import com.google.samples.apps.nowinandroid.bookmarks.goToBookmarksScreen
+import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
+import org.junit.Rule
+import org.junit.Test
+
+/**
+ * Baseline Profile of the "Bookmarks" screen
+ */
+class BookmarksBaselineProfile {
+    @get:Rule val baselineProfileRule = BaselineProfileRule()
+
+    @Test
+    fun generate() =
+        baselineProfileRule.collect(PACKAGE_NAME) {
+            startActivityAndAllowNotifications()
+
+            // Navigate to saved screen
+            goToBookmarksScreen()
+        }
+}

…elineprofile/BaselineProfileGenerator.kt …baselineprofile/ForYouBaselineProfile.ktbenchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/BaselineProfileGenerator.kt renamed to benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/ForYouBaselineProfile.kt benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/BaselineProfileGenerator.kt renamed to benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/ForYouBaselineProfile.kt

@@ -18,43 +18,27 @@ package com.google.samples.apps.nowinandroid.baselineprofile
 
 import androidx.benchmark.macro.junit4.BaselineProfileRule
 import com.google.samples.apps.nowinandroid.PACKAGE_NAME
-import com.google.samples.apps.nowinandroid.allowNotifications
-import com.google.samples.apps.nowinandroid.bookmarks.goToBookmarksScreen
 import com.google.samples.apps.nowinandroid.foryou.forYouScrollFeedDownUp
 import com.google.samples.apps.nowinandroid.foryou.forYouSelectTopics
 import com.google.samples.apps.nowinandroid.foryou.forYouWaitForContent
-import com.google.samples.apps.nowinandroid.interests.goToInterestsScreen
-import com.google.samples.apps.nowinandroid.interests.interestsScrollTopicsDownUp
+import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
 import org.junit.Rule
 import org.junit.Test
 
 /**
- * Generates a baseline profile which can be copied to `app/src/main/baseline-prof.txt`.
+ * Baseline Profile of the "For You" screen
  */
-class BaselineProfileGenerator {
+class ForYouBaselineProfile {
     @get:Rule val baselineProfileRule = BaselineProfileRule()
 
     @Test
     fun generate() =
         baselineProfileRule.collect(PACKAGE_NAME) {
-            // This block defines the app's critical user journey. Here we are interested in
-            // optimizing for app startup. But you can also navigate and scroll
-            // through your most important UI.
-            allowNotifications()
-            pressHome()
-            startActivityAndWait()
-            allowNotifications()
+            startActivityAndAllowNotifications()
 
             // Scroll the feed critical user journey
             forYouWaitForContent()
             forYouSelectTopics(true)
             forYouScrollFeedDownUp()
-
-            // Navigate to saved screen
-            goToBookmarksScreen()
-
-            // Navigate to interests screen
-            goToInterestsScreen()
-            interestsScrollTopicsDownUp()
         }
 }

benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/InterestsBaselineProfile.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.samples.apps.nowinandroid.baselineprofile
+
+import androidx.benchmark.macro.junit4.BaselineProfileRule
+import com.google.samples.apps.nowinandroid.PACKAGE_NAME
+import com.google.samples.apps.nowinandroid.interests.goToInterestsScreen
+import com.google.samples.apps.nowinandroid.interests.interestsScrollTopicsDownUp
+import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
+import org.junit.Rule
+import org.junit.Test
+
+/**
+ * Baseline Profile of the "Interests" screen
+ */
+class InterestsBaselineProfile {
+    @get:Rule val baselineProfileRule = BaselineProfileRule()
+
+    @Test
+    fun generate() =
+        baselineProfileRule.collect(PACKAGE_NAME) {
+            startActivityAndAllowNotifications()
+
+            // Navigate to interests screen
+            goToInterestsScreen()
+            interestsScrollTopicsDownUp()
+        }
+}

benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/baselineprofile/StartupBaselineProfile.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.samples.apps.nowinandroid.baselineprofile
+
+import androidx.benchmark.macro.junit4.BaselineProfileRule
+import com.google.samples.apps.nowinandroid.PACKAGE_NAME
+import com.google.samples.apps.nowinandroid.startActivityAndAllowNotifications
+import org.junit.Rule
+import org.junit.Test
+
+/**
+ * Baseline Profile for app startup. This profile also enables using [Dex Layout Optimizations](https://developer.android.com/topic/performance/baselineprofiles/dex-layout-optimizations)
+ * via the `includeInStartupProfile` parameter.
+ */
+class StartupBaselineProfile {
+    @get:Rule val baselineProfileRule = BaselineProfileRule()
+
+    @Test
+    fun generate() =
+        baselineProfileRule.collect(
+            PACKAGE_NAME,
+            includeInStartupProfile = true,
+        ) {
+            startActivityAndAllowNotifications()
+        }
+}

benchmarks/src/main/kotlin/com/google/samples/apps/nowinandroid/bookmarks/BookmarksActions.kt

@@ -18,13 +18,13 @@ package com.google.samples.apps.nowinandroid.bookmarks
 
 import androidx.benchmark.macro.MacrobenchmarkScope
 import androidx.test.uiautomator.By
-import androidx.test.uiautomator.Until
+import com.google.samples.apps.nowinandroid.waitForObjectOnTopAppBar
 
 fun MacrobenchmarkScope.goToBookmarksScreen() {
-    device.findObject(By.text("Saved")).click()
+    val savedSelector = By.text("Saved")
+    val savedButton = device.findObject(savedSelector)
+    savedButton.click()
     device.waitForIdle()
     // Wait until saved title are shown on screen
-    device.wait(Until.hasObject(By.res("niaTopAppBar")), 2_000)
-    val topAppBar = device.findObject(By.res("niaTopAppBar"))
-    topAppBar.wait(Until.hasObject(By.text("Saved")), 2_000)
+    waitForObjectOnTopAppBar(savedSelector)
 }