Skip to content

Commit a83efb8

Browse files
stefanosianostefanosiano
authored
Tests/android UI tests absolute (#2077)
1 parent e4a46aa commit a83efb8

File tree

13 files changed

+319
-97
lines changed

13 files changed

+319
-97
lines changed

‎.github/workflows/integration-tests-benchmarks.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: "Integration Tests - Benchmarks"
2+
on:
3+
push:
4+
branches:
5+
- main
6+
- release/**
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Benchmarks
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: Git checkout
16+
uses: actions/checkout@v2
17+
18+
- name: "Set up Java: 11"
19+
uses: actions/setup-java@v2
20+
with:
21+
java-version: "11"
22+
distribution: "adopt"
23+
24+
# Clean, build and release a test apk
25+
- name: Make assembleBenchmarks
26+
run: make assembleBenchmarks
27+
28+
# We stop gradle at the end to make sure the cache folders
29+
# don't contain any lock files and are free to be cached.
30+
- name: Make stop
31+
run: make stop
32+
33+
- name: Run Tests in SauceLab
34+
uses: saucelabs/saucectl-run-action@v1
35+
with:
36+
sauce-username: ${{ secrets.SAUCE_USERNAME }}
37+
sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
38+
config-file: .sauce/sentry-uitest-android-benchmark.yml

‎.github/workflows/integration-tests-ui.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: "Integration Tests - Ui tests"
2+
on:
3+
push:
4+
branches:
5+
- main
6+
- release/**
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Ui tests
12+
runs-on: ubuntu-latest
13+
14+
steps:
15+
- name: Git checkout
16+
uses: actions/checkout@v2
17+
18+
- name: "Set up Java: 11"
19+
uses: actions/setup-java@v2
20+
with:
21+
java-version: "11"
22+
distribution: "adopt"
23+
24+
# Clean, build and release a test apk
25+
- name: Make assembleUiTests
26+
run: make assembleUiTests
27+
28+
# We stop gradle at the end to make sure the cache folders
29+
# don't contain any lock files and are free to be cached.
30+
- name: Make stop
31+
run: make stop
32+
33+
- name: Run Tests in SauceLab
34+
uses: saucelabs/saucectl-run-action@v1
35+
with:
36+
sauce-username: ${{ secrets.SAUCE_USERNAME }}
37+
sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
38+
config-file: .sauce/sentry-uitest-android-ui.yml

‎.sauce/sentry-uitest-android-benchmark.yml‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,59 @@ apiVersion: v1alpha
22
kind: espresso
33
sauce:
44
region: us-west-1
5-
# Controls how many suites are executed at the same time (sauce test env only).
6-
concurrency: 1
5+
concurrency: 2
76
metadata:
87
name: Android benchmarks with Espresso
98
tags:
109
- benchmarks
1110
- android
11+
12+
defaults:
13+
timeout: 30m
14+
1215
espresso:
1316
app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk
1417
testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk
18+
1519
suites:
16-
name: "Android Benchmarks"
17-
devices:
18-
- name: "Google Pixel 2"
19-
platformVersion: 11
20-
- id: Google_Pixel_2_real_us
21-
testOptions:
22-
useTestOrchestrator: true
23-
# Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
20+
21+
# Devices are chosen so that there is a high-end and a low-end device for each api level
22+
- name: "Android 12 (api 31)"
23+
devices:
24+
- id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12)
25+
- id: Google_Pixel_3_12_real_us # Google Pixel 3 - api 31 (12)
26+
testOptions:
27+
useTestOrchestrator: true
28+
29+
- name: "Android 11 (api 30)"
30+
devices:
31+
- id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11)
32+
- id: Google_Pixel_2_real_us # Google Pixel 2 - api 30 (11)
33+
testOptions:
34+
useTestOrchestrator: true
35+
36+
# Commenting for the moment, due to the error "Cannot install test-services-1.4.1.apk on device" on low Android versions
37+
# - name: "Android 5 (api 22)"
38+
# devices:
39+
# - id: Amazon_Kindle_Fire_HD_8_real_us # Amazon Kindle Fire HD 8 - api 22 (5.1.1)
40+
# testOptions:
41+
# useTestOrchestrator: true
42+
43+
# - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
44+
# - id: Motorola_Moto_G_Power_real_us # Motorola Moto G Power (2021) - api 29 (10)
45+
# - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9)
46+
# - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9)
47+
# - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0)
48+
# - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0)
49+
# - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0)
50+
# - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0)
51+
# - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1)
52+
# - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1)
53+
# - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0)
54+
# - id: LG_K10_real # LG K10 - api 24 (7.0)
55+
# - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1)
56+
# - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1)
57+
2458
artifacts:
2559
download:
2660
when: always

‎.sauce/sentry-uitest-android-end2end.yml‎ renamed to ‎.sauce/sentry-uitest-android-ui.yml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,13 @@ espresso:
1313
app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk
1414
testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk
1515
suites:
16-
name: "Android End2end"
17-
emulators:
18-
- name: "Android GoogleApi Emulator"
19-
orientation: portrait
20-
platformVersions:
21-
- "11.0"
22-
- "10.0"
23-
testOptions:
24-
useTestOrchestrator: true
16+
17+
- name: "Android 12 (api 31)"
18+
devices:
19+
- id: Samsung_Galaxy_S22_Ultra_5G_real_us # Samsung Galaxy S22 Ultra 5G - api 31 (12)
20+
testOptions:
21+
useTestOrchestrator: true
22+
2523
# Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
2624
artifacts:
2725
download:

‎Makefile‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
.PHONY: all clean compile dryRelease update stop checkFormat format api
1+
.PHONY: all clean compile dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease
22

33
all: stop clean checkFormat compile dryRelease
4+
assembleBenchmarks: stop clean assembleBenchmarkTestRelease
5+
assembleUiTests: stop clean assembleUiTestRelease
46

57
# deep clean
68
clean:
@@ -35,3 +37,13 @@ format:
3537
# Binary compatibility validator
3638
api:
3739
./gradlew apiDump
40+
41+
# Assemble release and Android test apk of the uitest-android-benchmark module
42+
assembleBenchmarkTestRelease:
43+
./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease
44+
./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release
45+
46+
# Assemble release and Android test apk of the uitest-android module
47+
assembleUiTestRelease:
48+
./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease
49+
./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release

‎sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ dependencies {
8888

8989
implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION))
9090

91-
implementation(projects.sentryAndroidIntegrationTests.sentryUitestAndroid)
9291
implementation(projects.sentryAndroid)
9392
implementation(Config.Libs.appCompat)
9493
implementation(Config.Libs.androidxCore)

‎sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package io.sentry.uitest.android.benchmark
2+
3+
import android.content.Context
4+
import android.view.Choreographer
5+
import androidx.test.core.app.ApplicationProvider
6+
import androidx.test.platform.app.InstrumentationRegistry
7+
import androidx.test.runner.AndroidJUnitRunner
8+
import kotlin.test.BeforeTest
9+
10+
abstract class BaseBenchmarkTest {
11+
12+
protected lateinit var runner: AndroidJUnitRunner
13+
protected lateinit var context: Context
14+
protected lateinit var choreographer: Choreographer
15+
16+
@BeforeTest
17+
fun baseSetUp() {
18+
runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner
19+
context = ApplicationProvider.getApplicationContext()
20+
context.cacheDir.deleteRecursively()
21+
// Must run on the main thread to get the main thread choreographer.
22+
runner.runOnMainSync {
23+
choreographer = Choreographer.getInstance()
24+
}
25+
}
26+
}

‎sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package io.sentry.uitest.android.benchmark
2+
3+
import androidx.lifecycle.Lifecycle
4+
import androidx.test.core.app.launchActivity
5+
import androidx.test.ext.junit.runners.AndroidJUnit4
6+
import io.sentry.Sentry
7+
import io.sentry.android.core.SentryAndroid
8+
import io.sentry.uitest.android.benchmark.util.BenchmarkOperation
9+
import org.junit.runner.RunWith
10+
import java.util.concurrent.TimeUnit
11+
import kotlin.test.Test
12+
import kotlin.test.assertTrue
13+
14+
@RunWith(AndroidJUnit4::class)
15+
class SdkBenchmarkTest : BaseBenchmarkTest() {
16+
17+
@Test
18+
fun benchmarkSdkInit() {
19+
20+
// We compare starting an activity with and without the sdk init, to measure its impact on startup time.
21+
val opNoSdk = getOperation()
22+
val opSimpleSdk = getOperation {
23+
SentryAndroid.init(context) {
24+
it.dsn = "https://key@host/proj"
25+
}
26+
}
27+
val opNoSdk2 = getOperation()
28+
val opPerfProfilingSdk = getOperation {
29+
SentryAndroid.init(context) {
30+
it.dsn = "https://key@host/proj"
31+
it.isProfilingEnabled = true
32+
it.tracesSampleRate = 1.0
33+
}
34+
}
35+
val simpleSdkResult = BenchmarkOperation.compare(opNoSdk, "No Sdk", opSimpleSdk, "Simple Sdk")
36+
val perfProfilingSdkResult = BenchmarkOperation.compare(opNoSdk2, "No Sdk", opPerfProfilingSdk, "Sdk with perf and profiling")
37+
38+
val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(100)
39+
assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold)
40+
assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100)
41+
assertTrue(perfProfilingSdkResult.durationIncreaseNanos in simpleSdkResult.durationIncreaseNanos..maxDurationThreshold)
42+
assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in simpleSdkResult.cpuTimeIncreaseMillis..100)
43+
}
44+
45+
private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation(
46+
choreographer,
47+
op = {
48+
runner.runOnMainSync {
49+
init?.invoke()
50+
}
51+
val benchmarkScenario = launchActivity<BenchmarkActivity>()
52+
benchmarkScenario.moveToState(Lifecycle.State.DESTROYED)
53+
},
54+
after = {
55+
Sentry.close()
56+
}
57+
)
58+
}

0 commit comments

Comments
 (0)