Update benchmark cpu approach (#2473)

* updated benchmark devices
* added sustained performance mode on profiling benchmarks, but not on sdk init
* changed cpu time reading from Process.getElapsedCpuTime() to reading /proc/self/stat file
* reverted threshold from 5.5% to 5%
* removed absolute time check in SdkBenchmarkTest
* converted cpu time from milliseconds to nanoseconds

Author: stefanosiano

.sauce/sentry-uitest-android-benchmark.yml

@@ -25,23 +25,23 @@ suites:
       useTestOrchestrator: true
     devices:
       - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) - high end
-      - id: Google_Pixel_6a_real_us # Google Pixel 6a - api 31 (12) - low end
+      - id: Google_Pixel_5_12_real_us # Google Pixel 5 - api 31 (12) - low end
 
   - name: "Android 11 (api 30)"
     testOptions:
       clearPackageData: true
       useTestOrchestrator: true
     devices:
       - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11) - high end
-      - id: Google_Pixel_4_real_us # Google Pixel 4 - api 30 (11) - mid end
+      - id: Samsung_Galaxy_A71_5G_real_us # Samsung Galaxy A71 5G - api 30 (11) - mid end
       - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) - low end
 
   - name: "Android 10 (api 29)"
     testOptions:
       clearPackageData: true
       useTestOrchestrator: true
     devices:
-      - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
+      - id: Google_Pixel_3a_XL_real # Google Pixel 3a XL - api 29 (10)
       - id: Nokia_7_1_real_us # Nokia 7.1 - api 29 (10)
 
 # At the time of writing (July, 4, 2022), the market share per android version is:

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt

@@ -1,5 +1,6 @@
 package io.sentry.uitest.android.benchmark
 
+import android.os.Bundle
 import androidx.lifecycle.Lifecycle
 import androidx.test.core.app.launchActivity
 import androidx.test.ext.junit.runners.AndroidJUnit4
@@ -39,11 +40,8 @@ class SdkBenchmarkTest : BaseBenchmarkTest() {
         val perfProfilingSdkResult = perfProfilingSdkResults.getSummaryResult()
         perfProfilingSdkResult.printResults()
 
-        val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(250)
-        assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold)
-        assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100)
-        assertTrue(perfProfilingSdkResult.durationIncreaseNanos in 0..maxDurationThreshold)
-        assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in 0..100)
+        assertTrue(simpleSdkResult.cpuTimeIncreaseNanos in 0..TimeUnit.MILLISECONDS.toNanos(100))
+        assertTrue(perfProfilingSdkResult.cpuTimeIncreaseNanos in 0..TimeUnit.MILLISECONDS.toNanos(100))
     }
 
     private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation(
@@ -52,7 +50,9 @@ class SdkBenchmarkTest : BaseBenchmarkTest() {
             runner.runOnMainSync {
                 init?.invoke()
             }
-            val benchmarkScenario = launchActivity<BenchmarkActivity>()
+            val benchmarkScenario = launchActivity<BenchmarkActivity>(
+                activityOptions = Bundle().apply { putBoolean(BenchmarkActivity.EXTRA_SUSTAINED_PERFORMANCE_MODE, false) }
+            )
             benchmarkScenario.moveToState(Lifecycle.State.DESTROYED)
         },
         after = {

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt

@@ -1,5 +1,6 @@
 package io.sentry.uitest.android.benchmark
 
+import android.os.Bundle
 import androidx.lifecycle.Lifecycle
 import androidx.test.core.app.launchActivity
 import androidx.test.espresso.Espresso
@@ -89,7 +90,7 @@ class SentryBenchmarkTest : BaseBenchmarkTest() {
         comparisonResult.printResults()
 
         // Currently we just want to assert the cpu overhead
-        assertTrue(comparisonResult.cpuTimeIncreasePercentage in 0F..5.5F)
+        assertTrue(comparisonResult.cpuTimeIncreasePercentage in 0F..5F)
     }
 
     /**
@@ -99,7 +100,9 @@ class SentryBenchmarkTest : BaseBenchmarkTest() {
     private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction? = { null }): () -> Unit = {
         var transaction: ITransaction? = null
         // Launch the sentry-uitest-android-benchmark activity
-        val benchmarkScenario = launchActivity<BenchmarkActivity>()
+        val benchmarkScenario = launchActivity<BenchmarkActivity>(
+            activityOptions = Bundle().apply { putBoolean(BenchmarkActivity.EXTRA_SUSTAINED_PERFORMANCE_MODE, true) }
+        )
         // Starts a transaction (it can be null, but we still runOnMainSync to make operations as similar as possible)
         runner.runOnMainSync {
             transaction = transactionBuilder()

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkComparisonResult.kt

@@ -70,7 +70,7 @@ internal data class BenchmarkComparisonResult(
             println(
                 "$prefix CPU time overhead, over $cores cores: %.2f%% (%d ms)".format(
                     cpuTimeIncreasePercentages[index],
-                    cpuTimeIncreases[index]
+                    TimeUnit.NANOSECONDS.toMillis(cpuTimeIncreases[index])
                 )
             )
 
@@ -111,10 +111,10 @@ internal data class BenchmarkComparisonResult(
 /** Result of the [BenchmarkOperation] comparison. */
 internal data class BenchmarkSummaryResult(
     /**
-     * Increase of cpu time in milliseconds.
+     * Increase of cpu time in nanoseconds.
      * It has no direct impact on performance of the app, but it has on battery usage, as the cpu is 'awaken' longer.
      */
-    val cpuTimeIncreaseMillis: Long,
+    val cpuTimeIncreaseNanos: Long,
     /** Increase of cpu time in percentage. */
     val cpuTimeIncreasePercentage: Double,
     /**
@@ -146,7 +146,7 @@ internal data class BenchmarkSummaryResult(
                 TimeUnit.NANOSECONDS.toMillis(durationIncreaseNanos)
             )
         )
-        println("CPU time overhead: %.2f%% (%d ms)".format(cpuTimeIncreasePercentage, cpuTimeIncreaseMillis))
+        println("CPU time overhead: %.2f%% (%d ms)".format(cpuTimeIncreasePercentage, cpuTimeIncreaseNanos))
         println("FPS decrease: %.2f%% (%d fps)".format(fpsDecreasePercentage, fpsDecrease))
         println("Frame drop increase: %.2f%% (%.2f)".format(droppedFramesIncreasePercentage, droppedFramesIncrease))
     }

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt

@@ -1,8 +1,10 @@
 package io.sentry.uitest.android.benchmark.util
 
-import android.os.Process
 import android.os.SystemClock
+import android.system.Os
+import android.system.OsConstants
 import android.view.Choreographer
+import java.io.File
 import java.util.LinkedList
 import java.util.concurrent.TimeUnit
 
@@ -79,7 +81,7 @@ internal class BenchmarkOperation(
     }
 
     private var lastFrameTimeNanos: Long = 0
-    private val cpuDurationMillisList: MutableList<Long> = LinkedList()
+    private val cpuDurationNanosList: MutableList<Long> = LinkedList()
     private val droppedFramesList: MutableList<Double> = LinkedList()
     private val durationNanosList: MutableList<Long> = LinkedList()
     private val fpsList: MutableList<Int> = LinkedList()
@@ -99,15 +101,15 @@ internal class BenchmarkOperation(
         frameCallback.setup(refreshRate)
 
         val startRealtimeNs = SystemClock.elapsedRealtimeNanos()
-        val startCpuTimeMs = Process.getElapsedCpuTime()
+        val startCpuTimeMs = readProcessorTimeNanos()
         lastFrameTimeNanos = startRealtimeNs
 
         choreographer.postFrameCallback(frameCallback)
         op()
         choreographer.removeFrameCallback(frameCallback)
 
         val durationNanos = SystemClock.elapsedRealtimeNanos() - startRealtimeNs
-        cpuDurationMillisList.add(Process.getElapsedCpuTime() - startCpuTimeMs)
+        cpuDurationNanosList.add(readProcessorTimeNanos() - startCpuTimeMs)
         durationNanosList.add(durationNanos)
         droppedFramesList.add(frameCallback.droppedFrames)
         // fps = counted frames per seconds converted into frames per nanoseconds, divided by duration in nanoseconds
@@ -118,10 +120,26 @@ internal class BenchmarkOperation(
         isolate()
     }
 
+    private fun readProcessorTimeNanos(): Long {
+        val clockSpeedHz = Os.sysconf(OsConstants._SC_CLK_TCK)
+//        val numCores = Os.sysconf(OsConstants._SC_NPROCESSORS_CONF)
+        val nanosecondsPerClockTick = 1_000_000_000 / clockSpeedHz.toDouble()
+        val selfStat = File("/proc/self/stat")
+        val stats = selfStat.readText().trim().split("[\n\t\r ]".toRegex())
+        if (stats.isNotEmpty()) {
+            val uTime = stats[13].toLong()
+            val sTime = stats[14].toLong()
+            val cuTime = stats[15].toLong()
+            val csTime = stats[16].toLong()
+            return ((uTime + sTime + cuTime + csTime) * nanosecondsPerClockTick).toLong()
+        }
+        return 0
+    }
+
     /** Return the [BenchmarkOperationComparable] for the operation. */
     private fun getResult(operationName: String): BenchmarkOperationComparable {
         return BenchmarkOperationComparable(
-            cpuDurationMillisList,
+            cpuDurationNanosList,
             droppedFramesList,
             durationNanosList,
             fpsList,

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationComparable.kt

@@ -4,7 +4,7 @@ import java.util.concurrent.TimeUnit
 
 /** Stores the results of a single run of [BenchmarkOperation]. */
 internal data class BenchmarkOperationComparable(
-    val cpuTimeMillis: List<Long>,
+    val cpuTimeNanos: List<Long>,
     val droppedFrames: List<Double>,
     val durationNanos: List<Long>,
     val fps: List<Int>,
@@ -15,7 +15,7 @@ internal data class BenchmarkOperationComparable(
         val cores = Runtime.getRuntime().availableProcessors()
         val durationIncreaseNanos = ArrayList<Long>()
         val durationIncreasePercentage = ArrayList<Double>()
-        val cpuTimeIncreaseMillis = ArrayList<Long>()
+        val cpuTimeIncreaseNanos = ArrayList<Long>()
         val cpuTimeOverheadPercentage = ArrayList<Double>()
         val fpsDecrease = ArrayList<Int>()
         val fpsDecreasePercentage = ArrayList<Double>()
@@ -29,8 +29,8 @@ internal data class BenchmarkOperationComparable(
 
             // Measure average cpu time
             // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only.
-            cpuTimeIncreaseMillis.add((cpuTimeMillis[index] - other.cpuTimeMillis[index]) / cores)
-            cpuTimeOverheadPercentage.add(cpuTimeIncreaseMillis[index] * 100.0 / other.cpuTimeMillis[index])
+            cpuTimeIncreaseNanos.add((cpuTimeNanos[index] - other.cpuTimeNanos[index]) / cores)
+            cpuTimeOverheadPercentage.add(cpuTimeIncreaseNanos[index] * 100.0 / other.cpuTimeNanos[index])
 
             // Measure average fps
             fpsDecrease.add(other.fps[index] - fps[index])
@@ -50,9 +50,9 @@ internal data class BenchmarkOperationComparable(
             cores,
             operationName,
             other.operationName,
-            cpuTimeMillis,
-            other.cpuTimeMillis,
-            cpuTimeIncreaseMillis,
+            cpuTimeNanos,
+            other.cpuTimeNanos,
+            cpuTimeIncreaseNanos,
             cpuTimeOverheadPercentage,
             droppedFrames,
             other.droppedFrames,

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt

@@ -12,14 +12,15 @@ import java.util.concurrent.Executors
 
 /** A simple activity with a list of bitmaps. */
 class BenchmarkActivity : AppCompatActivity() {
-
     companion object {
 
         /** The activity will set this when scrolling. */
         val scrollingIdlingResource = CountingIdlingResource("sentry-uitest-android-benchmark-activityScrolling")
 
         /** The refresh rate of the device, set on activity create. */
         var refreshRate: Float? = null
+
+        internal const val EXTRA_SUSTAINED_PERFORMANCE_MODE = "EXTRA_SUSTAINED_PERFORMANCE_MODE"
     }
 
     /**
@@ -35,6 +36,12 @@ class BenchmarkActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
 
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N &&
+            savedInstanceState?.getBoolean(EXTRA_SUSTAINED_PERFORMANCE_MODE) == true
+        ) {
+            window.setSustainedPerformanceMode(true)
+        }
+
         refreshRate = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
             display?.refreshRate
         } else {