Skip to content

Commit df710de

Browse files
steve-the-edwardssteve-the-edwards
committed
Add in Latency Benchmark for Workflow
Measure time from input event to Rendering produced, and time from Rendering passed to UI Layer to frame rendered.
1 parent ffba686 commit df710de

File tree

10 files changed

+386
-52
lines changed

10 files changed

+386
-52
lines changed

‎benchmarks/performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryBenchmarks.kt

Lines changed: 101 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package com.squareup.benchmarks.performance.complex.poetry.benchmark
22

33
import android.content.Context
4+
import android.content.Intent
45
import androidx.benchmark.macro.BaselineProfileMode
56
import androidx.benchmark.macro.CompilationMode
67
import androidx.benchmark.macro.ExperimentalMetricApi
7-
import androidx.benchmark.macro.FrameTimingMetric
88
import androidx.benchmark.macro.StartupMode
99
import androidx.benchmark.macro.StartupTimingMetric
1010
import androidx.benchmark.macro.TraceSectionMetric
@@ -13,8 +13,11 @@ import androidx.test.core.app.ApplicationProvider
1313
import androidx.test.ext.junit.runners.AndroidJUnit4
1414
import androidx.test.platform.app.InstrumentationRegistry
1515
import androidx.test.uiautomator.UiDevice
16+
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity
17+
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion
1618
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.PerformanceTracingInterceptor
1719
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.PerformanceTracingInterceptor.Companion.NODES_TO_TRACE
20+
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.SimulatedPerfConfig
1821
import com.squareup.benchmarks.performance.complex.poetry.robots.landscapeOrientation
1922
import com.squareup.benchmarks.performance.complex.poetry.robots.openRavenAndNavigate
2023
import org.junit.Before
@@ -27,14 +30,13 @@ import org.junit.runner.RunWith
2730
* info.
2831
*
2932
* [benchmarkStartup] will measure startup times with different compilation modes.
33+
* The above can be run as tests using Full, Partial, or No aot compiling on the app.
3034
*
31-
* [benchmarkTraceSections] will measure the trace timings instrumented via the
35+
* [benchmarkNodeAndRenderPassTraceSections] will measure the trace timings instrumented via the
3236
* [PerformanceTracingInterceptor] installed by default in the Workflow tree.
3337
*
34-
* [benchmarkFrameTiming] measures frame timing but it is still WIP and not useful in its current
35-
* form as there is not enough scrolling/animation in the scenario.
36-
*
37-
* The above can be run as tests using Full, Partial, or No aot compiling on the app.
38+
* [benchmarkLatencyTraceSections] will measure the time between a UI event to producing a
39+
* Rendering and the time between the Rendering and the Choreographer rendering the frame.
3840
*/
3941
@RunWith(AndroidJUnit4::class)
4042
class ComplexPoetryBenchmarks {
@@ -78,16 +80,16 @@ class ComplexPoetryBenchmarks {
7880
}
7981

8082
/**
81-
* This is a LONG test. Searching and pulling form the perfetto trace after each
83+
* This is a LONG test. Searching and pulling form the Perfetto trace after each
8284
* iteration takes a long time. This test with 20 iterations runs for 1 hr 12 m on
8385
* a Nexus 6.
8486
*/
85-
@Test fun benchmarkTraceSectionsFullAOT() {
86-
benchmarkTraceSections(CompilationMode.Full())
87+
@Test fun benchmarkNodeAndRenderPassTraceSectionsFullAot() {
88+
benchmarkNodeAndRenderPassTraceSections(CompilationMode.Full())
8789
}
8890

8991
@OptIn(ExperimentalMetricApi::class)
90-
private fun benchmarkTraceSections(compilationMode: CompilationMode) {
92+
private fun benchmarkNodeAndRenderPassTraceSections(compilationMode: CompilationMode) {
9193
val traceMetricsList = NODES_TO_TRACE.flatMap { node ->
9294
List(RENDER_PASSES + 1) { i ->
9395
val passNumber = i.toString()
@@ -110,50 +112,121 @@ class ComplexPoetryBenchmarks {
110112
compilationMode = compilationMode,
111113
setupBlock = {
112114
pressHome()
115+
device.landscapeOrientation()
113116
}
114117
) {
118+
startActivityAndWait{
119+
val renderPassConfig = SimulatedPerfConfig(
120+
isComplex = true,
121+
complexityDelay = 200L,
122+
useInitializingState = true,
123+
traceRenderingPasses = true,
124+
traceLatency = false
125+
)
126+
it.putExtra(PerformancePoetryActivity.PERF_CONFIG_EXTRA, renderPassConfig)
127+
}
115128
device.landscapeOrientation()
116-
// Use default performance config for now, so no need to customize intent.
117-
startActivityAndWait()
118129

119130
device.openRavenAndNavigate()
120131
}
121132
}
122133

123-
@Test fun benchmarkFrameTimingNoCompilation() {
124-
benchmarkFrameTiming(CompilationMode.None())
125-
}
126-
127-
@Test fun benchmarkFrameTimingPartialAOTWithProfile() {
128-
benchmarkFrameTiming(CompilationMode.Partial(baselineProfileMode = BaselineProfileMode.Require))
129-
}
130-
131-
@Test fun benchmarkFrameTimingFullAOT() {
132-
benchmarkFrameTiming(CompilationMode.Full())
134+
/**
135+
* Another LONG test.
136+
*/
137+
@Test fun benchmarkLatencyTraceSectionsFullAot() {
138+
benchmarkLatencyTraceSections(CompilationMode.Full())
133139
}
134140

141+
/**
142+
* This tests is focused on two different measurements:
143+
*
144+
* Frame-Latency-N: is the trace between passing the Rendering to the view layer and the
145+
* 'post frame rendered callback' for the Nth frame in the scenario. In other words, this traces
146+
* the time it takes from a Rendering produced by Workflow to process through the Workflow UI
147+
* layer and then be rendered in the next frame.
148+
*
149+
* XScreen-onY-Z: is the time between the execution of event handler 'onY' and the production of
150+
* the next root Rendering by Workflow for the Zth instance of the 'onY' handler on X Screen.
151+
* In other words, this measures the time Workflow takes in processing a UI event into a new
152+
* Rendering. This will be similar to the render pass traced above, but more comprehensive to
153+
* include all of the event handling time.
154+
*/
135155
@OptIn(ExperimentalMetricApi::class)
136-
private fun benchmarkFrameTiming(compilationMode: CompilationMode) {
156+
fun benchmarkLatencyTraceSections(compilationMode: CompilationMode) {
137157
benchmarkRule.measureRepeated(
138158
packageName = PACKAGE_NAME,
139-
metrics = listOf(FrameTimingMetric()),
140-
iterations = 1,
159+
metrics = LATENCY_TRACE_SECTIONS.map { TraceSectionMetric(it) },
160+
iterations = 20,
141161
startupMode = StartupMode.WARM,
142162
compilationMode = compilationMode,
143163
setupBlock = {
144164
pressHome()
165+
device.landscapeOrientation()
145166
}
146167
) {
147-
startActivityAndWait()
168+
startActivityAndWait{
169+
val renderPassConfig = SimulatedPerfConfig(
170+
isComplex = true,
171+
complexityDelay = 200L,
172+
useInitializingState = true,
173+
traceRenderingPasses = false,
174+
traceLatency = true
175+
)
176+
it.putExtra(PerformancePoetryActivity.PERF_CONFIG_EXTRA, renderPassConfig)
177+
}
178+
device.landscapeOrientation()
148179

149-
// N.B. This is *not* a good scenario to measure frame timing as there isn't much scrolling
150-
// or animation involved. This benchmark needs another app scenario.
151180
device.openRavenAndNavigate()
152181
}
153182
}
154183

155184
companion object {
156185
const val RENDER_PASSES = 61
157186
const val PACKAGE_NAME = "com.squareup.benchmarks.performance.complex.poetry"
187+
188+
val LATENCY_TRACE_SECTIONS = listOf(
189+
"PoemListScreen-onPoemSelected(2)-1 ",
190+
"StanzaListScreen-onStanzaSelected(4)-1 ",
191+
"StanzaScreen-onGoForth-1 ",
192+
"StanzaScreen-onGoForth-2 ",
193+
"StanzaScreen-onGoForth-3 ",
194+
"StanzaScreen-onGoForth-4 ",
195+
"StanzaScreen-onGoForth-5 ",
196+
"StanzaScreen-onGoBack-1 ",
197+
"StanzaScreen-onGoBack-2 ",
198+
"StanzaScreen-onGoBack-3 ",
199+
"StanzaScreen-onGoBack-4 ",
200+
"StanzaScreen-onGoBack-5 ",
201+
"StanzaListScreen-onExit-1 ",
202+
"Frame-Latency-00 ",
203+
"Frame-Latency-01 ",
204+
"Frame-Latency-02 ",
205+
"Frame-Latency-03 ",
206+
"Frame-Latency-04 ",
207+
"Frame-Latency-05 ",
208+
"Frame-Latency-06 ",
209+
"Frame-Latency-07 ",
210+
"Frame-Latency-08 ",
211+
"Frame-Latency-09 ",
212+
"Frame-Latency-10 ",
213+
"Frame-Latency-11 ",
214+
"Frame-Latency-12 ",
215+
"Frame-Latency-13 ",
216+
"Frame-Latency-14 ",
217+
"Frame-Latency-15 ",
218+
"Frame-Latency-16 ",
219+
"Frame-Latency-17 ",
220+
"Frame-Latency-18 ",
221+
"Frame-Latency-19 ",
222+
"Frame-Latency-20 ",
223+
"Frame-Latency-21 ",
224+
"Frame-Latency-22 ",
225+
"Frame-Latency-23 ",
226+
"Frame-Latency-24 ",
227+
"Frame-Latency-25 ",
228+
"Frame-Latency-26 ",
229+
"Frame-Latency-27 ",
230+
)
158231
}
159232
}

‎benchmarks/performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryResults.txt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,3 +187,56 @@ ComplexPoetryBenchmarks_benchmarkTraceSectionsFullAOT
187187
060_Render_Pass_Ms min 0.0, median 0.0, max 0.1
188188
060_Render_Pass_Node_PerformancePoemsBrowserWorkflow_Ms min 0.0, median 0.0, max 0.0
189189
Traces: Iteration 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
190+
191+
192+
May 6, 2022
193+
194+
> Task :benchmarks:performance-poetry:complex-benchmark:connectedBenchmarkAndroidTest
195+
Starting 1 tests on Pixel 6 - 12
196+
Connected to process 8330 on device 'google-pixel_6-1C041FDF60017G'.
197+
198+
ComplexPoetryBenchmarks_benchmarkLatencySections
199+
Frame-Latency-00 Ms min 41.8, median 45.7, max 57.9
200+
Frame-Latency-01 Ms min 68.2, median 96.1, max 125.8
201+
Frame-Latency-02 Ms min 5.1, median 14.0, max 19.0
202+
Frame-Latency-03 Ms min 15.0, median 23.6, max 35.9
203+
Frame-Latency-04 Ms min 59.0, median 65.9, max 79.4
204+
Frame-Latency-05 Ms min 16.8, median 28.3, max 40.3
205+
Frame-Latency-06 Ms min 58.6, median 70.0, max 78.5
206+
Frame-Latency-07 Ms min 24.0, median 29.1, max 35.9
207+
Frame-Latency-08 Ms min 58.8, median 67.2, max 73.0
208+
Frame-Latency-09 Ms min 26.2, median 32.7, max 39.1
209+
Frame-Latency-10 Ms min 58.5, median 67.1, max 79.0
210+
Frame-Latency-11 Ms min 17.7, median 32.5, max 38.3
211+
Frame-Latency-12 Ms min 53.8, median 70.8, max 79.5
212+
Frame-Latency-13 Ms min 23.4, median 30.9, max 35.3
213+
Frame-Latency-14 Ms min 57.9, median 67.0, max 74.9
214+
Frame-Latency-15 Ms min 22.7, median 30.7, max 39.8
215+
Frame-Latency-16 Ms min 57.3, median 66.6, max 72.3
216+
Frame-Latency-17 Ms min 27.6, median 32.9, max 36.8
217+
Frame-Latency-18 Ms min 54.1, median 65.6, max 74.0
218+
Frame-Latency-19 Ms min 17.5, median 32.1, max 39.2
219+
Frame-Latency-20 Ms min 55.6, median 63.5, max 75.6
220+
Frame-Latency-21 Ms min 25.7, median 31.6, max 39.3
221+
Frame-Latency-22 Ms min 55.6, median 64.5, max 75.4
222+
Frame-Latency-23 Ms min 23.3, median 29.6, max 34.4
223+
Frame-Latency-24 Ms min 57.7, median 67.0, max 72.8
224+
Frame-Latency-25 Ms min 25.7, median 31.6, max 36.3
225+
Frame-Latency-26 Ms min 35.1, median 41.6, max 68.9
226+
Frame-Latency-27 Ms min 18.4, median 24.3, max 33.7
227+
PoemListScreen-onPoemSelected(2)-1 Ms min 1.5, median 1.6, max 3.8
228+
StanzaListScreen-onExit-1 Ms min 0.6, median 0.7, max 1.1
229+
StanzaListScreen-onStanzaSelected(4)-1 Ms min 0.7, median 1.0, max 1.3
230+
StanzaScreen-onGoBack-1 Ms min 0.7, median 0.8, max 1.1
231+
StanzaScreen-onGoBack-2 Ms min 0.6, median 0.8, max 1.7
232+
StanzaScreen-onGoBack-3 Ms min 0.7, median 0.8, max 1.5
233+
StanzaScreen-onGoBack-4 Ms min 0.7, median 0.8, max 1.1
234+
StanzaScreen-onGoBack-5 Ms min 0.6, median 0.8, max 1.4
235+
StanzaScreen-onGoForth-1 Ms min 0.9, median 1.2, max 1.7
236+
StanzaScreen-onGoForth-2 Ms min 0.6, median 0.9, max 1.7
237+
StanzaScreen-onGoForth-3 Ms min 0.6, median 0.8, max 1.2
238+
StanzaScreen-onGoForth-4 Ms min 0.7, median 0.8, max 1.1
239+
StanzaScreen-onGoForth-5 Ms min 0.6, median 0.8, max 1.3
240+
Traces: Iteration 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
241+
242+
> Task :benchmarks:performance-poetry:complex-benchmark:connectedBenchmarkAndroidTest

‎benchmarks/performance-poetry/complex-poetry/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ dependencies {
6060
implementation(libs.androidx.recyclerview)
6161
implementation(libs.androidx.test.uiautomator)
6262
implementation(libs.androidx.tracing.ktx)
63+
implementation(libs.squareup.tart)
6364
implementation(libs.timber)
6465

6566
androidTestImplementation(project(":workflow-ui:internal-testing-android"))

‎benchmarks/performance-poetry/complex-poetry/src/main/java/com/squareup/benchmarks/performance/complex/poetry/PerformancePoemWorkflow.kt

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemWorkflo
99
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemWorkflow.State.Initializing
1010
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemWorkflow.State.Selected
1111
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.SimulatedPerfConfig
12+
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.trace
1213
import com.squareup.benchmarks.performance.complex.poetry.views.BlankScreen
1314
import com.squareup.sample.container.overviewdetail.OverviewDetailScreen
1415
import com.squareup.sample.poetry.PoemWorkflow
@@ -55,7 +56,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
5556
*/
5657
class PerformancePoemWorkflow(
5758
private val simulatedPerfConfig: SimulatedPerfConfig = SimulatedPerfConfig.NO_SIMULATED_PERF,
58-
private val isLoading: MutableStateFlow<Boolean>
59+
private val isLoading: MutableStateFlow<Boolean>,
5960
) : PoemWorkflow, StatefulWorkflow<Poem, State, ClosePoem, OverviewDetailScreen>() {
6061

6162
sealed class State {
@@ -132,6 +133,8 @@ class PerformancePoemWorkflow(
132133
context.renderChild(StanzaWorkflow, Props(renderProps, index), "$index") {
133134
noAction()
134135
}
136+
}.map { originalStanzaScreen ->
137+
originalStanzaScreen.trace()
135138
}
136139

137140
val visibleStanza =
@@ -146,7 +149,7 @@ class PerformancePoemWorkflow(
146149
ShowPreviousStanza -> SelectPrevious(simulatedPerfConfig)
147150
ShowNextStanza -> SelectNext(simulatedPerfConfig)
148151
}
149-
}
152+
}.trace()
150153
}
151154

152155
val stackedStanzas = visibleStanza?.let {
@@ -161,6 +164,7 @@ class PerformancePoemWorkflow(
161164
HandleStanzaListOutput(simulatedPerfConfig, selected)
162165
}
163166
.copy(selection = stanzaIndex)
167+
.trace()
164168

165169
stackedStanzas
166170
?.let {

‎benchmarks/performance-poetry/complex-poetry/src/main/java/com/squareup/benchmarks/performance/complex/poetry/PerformancePoemsBrowserWorkflow.kt

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemsBrowse
66
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemsBrowserWorkflow.State.NoSelection
77
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoemsBrowserWorkflow.State.Selected
88
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.SimulatedPerfConfig
9+
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.trace
910
import com.squareup.benchmarks.performance.complex.poetry.views.BlankScreen
1011
import com.squareup.sample.container.overviewdetail.OverviewDetailScreen
1112
import com.squareup.sample.poetry.PoemListScreen.Companion.NO_POEM_SELECTED
@@ -43,7 +44,7 @@ import kotlinx.coroutines.flow.MutableStateFlow
4344
class PerformancePoemsBrowserWorkflow(
4445
private val simulatedPerfConfig: SimulatedPerfConfig,
4546
private val poemWorkflow: PoemWorkflow,
46-
private val isLoading: MutableStateFlow<Boolean>
47+
private val isLoading: MutableStateFlow<Boolean>,
4748
) :
4849
PoemsBrowserWorkflow,
4950
StatefulWorkflow<List<Poem>, State, Unit, OverviewDetailScreen>() {
@@ -95,7 +96,7 @@ class PerformancePoemsBrowserWorkflow(
9596
is NoSelection -> {
9697
return OverviewDetailScreen(
9798
overviewRendering = BackStackScreen(
98-
poemListRendering.copy(selection = NO_POEM_SELECTED)
99+
poemListRendering.copy(selection = NO_POEM_SELECTED).trace()
99100
)
100101
)
101102
}
@@ -121,7 +122,7 @@ class PerformancePoemsBrowserWorkflow(
121122
}
122123
var poems = OverviewDetailScreen(
123124
overviewRendering = BackStackScreen(
124-
poemListRendering.copy(selection = renderState.payload)
125+
poemListRendering.copy(selection = renderState.payload).trace()
125126
)
126127
)
127128
if (renderState.payload != NO_POEM_SELECTED) {
@@ -136,7 +137,7 @@ class PerformancePoemsBrowserWorkflow(
136137
is Selected -> {
137138
val poems = OverviewDetailScreen(
138139
overviewRendering = BackStackScreen(
139-
poemListRendering.copy(selection = renderState.poemIndex)
140+
poemListRendering.copy(selection = renderState.poemIndex).trace()
140141
)
141142
)
142143
val poem: OverviewDetailScreen = context.renderChild(

0 commit comments

Comments
 (0)