Add snapshot testing framework (#400)

* work in progress

* work in progress - add ui test to travis

* wip - fix webview crash on android 5.1.1

* wip - try fix sdcard issue

* wip - try fix sdcard issue

* try snapshotting

* test verify task

* travis - use new sdkmanager and avdmanager

* new image for test, other fixes

* verify with new image

* broke snapshot test intentionally

* remove unnecessary code

* try to run unit tests while emulator is loading to reduce build time

* add README.md and small change in travis

* Little change in README

* Add snapshot test for Rich Interstitial message

* Increase snapshot WebView wait time

* Add Interstitial and WebInterstitial snapshot tests

Co-authored-by: Mayank Sanganeria <[email protected]>

Author: hborisoff

.travis.yml

@@ -4,6 +4,9 @@ env:
   global:
     - BUILD_API=29
     - BUILD_TOOLS=29.0.3
+    - ABI=x86
+    - EMU_API=22
+    - EMU_FLAVOR=default
     - ANDROID_HOME=/usr/local/android-sdk
     - TOOLS=${ANDROID_HOME}/tools
     # PATH order is important, the 'emulator' script exists in more than one place
@@ -21,19 +24,29 @@ licenses:
 before_install:
   - sudo wget "https://bouncycastle.org/download/bcprov-ext-jdk15on-165.jar" -O "${JAVA_HOME}/jre/lib/ext/bcprov-ext-jdk15on-165.jar"
   - sudo echo "security.provider.11=org.bouncycastle.jce.provider.BouncyCastleProvider" | sudo tee -a ${JAVA_HOME}/jre/lib/security/java.security
+  - python -m pip install Pillow --user # library for image manipulation in snapshot tests
 
 before_script:
-  # Install Android SDK
+  # Install Android SDK and run emulator
   - echo 'count=0' > /home/travis/.android/repositories.cfg # avoid harmless sdkmanager warning
   - echo y | sdkmanager "platform-tools" >/dev/null
   - echo y | sdkmanager "tools" >/dev/null
   - echo y | sdkmanager "build-tools;$BUILD_TOOLS" >/dev/null
+  - echo y | sdkmanager "platforms;android-$EMU_API" >/dev/null
   - echo y | sdkmanager "platforms;android-$BUILD_API" >/dev/null
   - echo y | sdkmanager "extras;android;m2repository" >/dev/null
+  - echo y | sdkmanager "system-images;android-$EMU_API;$EMU_FLAVOR;$ABI"
+  - echo no | avdmanager create avd --force -n test -k "system-images;android-$EMU_API;$EMU_FLAVOR;$ABI" -c 100M
+  - emulator -verbose -avd test -no-accel -no-snapshot -no-window -no-audio -camera-back none -camera-front none -selinux permissive -qemu -m 2048 &
 
 script:
   - ./gradlew assembleRelease testReleaseUnitTest
 
+  # wait emulator to come alive and start snapshot tests
+  - android-wait-for-emulator
+  - adb shell input keyevent 82 &
+  - ./gradlew verifyDebugAndroidTestScreenshotTest
+
 before_deploy:
   - ./Tools/verifyTag.sh
   - ./gradlew assembleRelease generatePomFileForAarPublication

AndroidSDKTests/README.md

@@ -0,0 +1,41 @@
+Snapshot testing
+------------
+
+This file shows how to execute and record the snapshot tests (macOS).
+
+Tool used for snapshot testing is [screenshot-tests-for-android](https://facebook.github.io/screenshot-tests-for-android/).
+
+Prerequisites
+------------
+
+#### Install Pillow (python library for images):
+
+    $ sudo -H python -m ensurepip
+    $ python -m pip install Pillow --user
+
+#### Install and run emulator:
+
+    $ $ANDROID_HOME/tools/bin/sdkmanager "system-images;android-22;default;x86"
+    $ echo no | $ANDROID_HOME/tools/bin/avdmanager create avd --force -n test -k "system-images;android-22;default;x86" -c 100M
+    $ $ANDROID_HOME/emulator/emulator -verbose -avd test -no-snapshot -no-audio -camera-back none -camera-front none -selinux permissive -qemu -m 2048 &
+
+Usage
+------------
+#### Executing tests
+
+    $ ./gradlew verifyDebugAndroidTestScreenshotTest
+
+If tests are failing compare snapshots:
+- your snapshots - AndroidSDKTests/build/screenshotsDebugAndroidTest
+- reference snapshots - AndroidSDKTests/screenshots
+
+You could use [ImageMagick](https://imagemagick.org/).
+
+    $ brew install imagemagick
+    $ magick left.png right.png difference.png
+
+#### Recording tests
+
+    $ ./gradlew recordDebugAndroidTestScreenshotTest
+
+Commit newly created PNG files located in AndroidSDKTests/screenshots.

AndroidSDKTests/build.gradle

@@ -1,5 +1,15 @@
 apply plugin: 'com.android.application'
 apply from: "../common-methods.gradle"
+apply plugin: 'com.facebook.testing.screenshot'
+
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'com.facebook.testing.screenshot:plugin:0.12.0'
+    }
+}
 
 android {
     compileSdkVersion COMPILE_SDK_VERSION
@@ -10,6 +20,8 @@ android {
         minSdkVersion MIN_SDK_VERSION
         versionCode 1
         versionName "1.0"
+
+        testInstrumentationRunner "com.leanplum.ScreenshotTestRunner"
     }
     buildTypes {
         release {
@@ -50,7 +62,7 @@ dependencies {
     implementation project(':AndroidSDKPush')
     implementation project(':AndroidSDKLocation')
     implementation project(':AndroidSDKFcm')
-    implementation "androidx.appcompat:appcompat:${APPCOMPAT_LIBRARY_VERSION}"
+    implementation "androidx.appcompat:appcompat:1.2.0-beta01" // WebView in Emulator with API22 is failing with exception because of appcompat version. In 1.2.0-beta01 it is not reproduced.
     implementation 'com.google.android.gms:play-services-location:17.0.0'
 
     // Dependencies used for unit tests.
@@ -66,6 +78,16 @@ dependencies {
     testImplementation 'org.powermock:powermock-api-mockito:1.6.6'
     testImplementation 'org.powermock:powermock-classloading-xstream:1.6.6'
     testImplementation 'org.bouncycastle:bcmail-jdk15on:1.65'
+
+    androidTestImplementation 'androidx.test.ext:junit:1.1.1'
+    androidTestImplementation 'androidx.test:runner:1.2.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
+    androidTestImplementation 'androidx.test:rules:1.2.0'
+    androidTestImplementation 'org.mockito:mockito-core:1.10.19'
+    androidTestImplementation "com.crittercism.dexmaker:dexmaker:1.4"
+    androidTestImplementation "com.crittercism.dexmaker:dexmaker-mockito:1.4"
+    androidTestImplementation "com.crittercism.dexmaker:dexmaker-dx:1.4"
+    implementation 'com.facebook.testing.screenshot:core:0.12.0'
 }
 
 task jarTests(type: Jar, dependsOn: "assembleDebugUnitTest") {

AndroidSDKTests/screenshots/centerPopup.png

@@ -0,0 +1,19 @@
+package com.leanplum;
+
+import android.os.Bundle;
+import androidx.test.runner.AndroidJUnitRunner;
+import com.facebook.testing.screenshot.ScreenshotRunner;
+
+public class ScreenshotTestRunner extends AndroidJUnitRunner {
+  @Override
+  public void onCreate(Bundle args) {
+    ScreenshotRunner.onCreate(this, args);
+    super.onCreate(args);
+  }
+
+  @Override
+  public void finish(int resultCode, Bundle results) {
+    ScreenshotRunner.onDestroy();
+    super.finish(resultCode, results);
+  }
+}

AndroidSDKTests/screenshots/interstitial.png

@@ -0,0 +1,45 @@
+package com.leanplum.messagetemplates;
+
+import android.app.Activity;
+import android.content.Context;
+import android.content.pm.PackageManager;
+import android.view.View;
+import androidx.annotation.NonNull;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.rule.ActivityTestRule;
+import com.facebook.testing.screenshot.Screenshot;
+import com.facebook.testing.screenshot.ViewHelpers;
+import com.leanplum.tests.MainActivity;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+@RunWith(AndroidJUnit4.class)
+public abstract class BaseSnapshotTest {
+  private static final int SNAPSHOT_WIDTH_DP = 480;
+  private static final int SNAPSHOT_HEIGHT_DP = 640;
+
+  @Rule
+  public ActivityTestRule<MainActivity> activityRule = new ActivityTestRule<>(MainActivity.class);
+
+  protected abstract String getSnapshotName();
+
+  protected Activity getMainActivity() {
+    return activityRule.getActivity();
+  }
+
+  protected String getApplicationName() {
+    PackageManager pm = getMainActivity().getPackageManager();
+    return getMainActivity().getApplicationInfo().loadLabel(pm).toString();
+  }
+
+  protected void setupView(@NonNull View view) {
+    ViewHelpers.setupView(view)
+        .setExactWidthDp(SNAPSHOT_WIDTH_DP)
+        .setExactHeightDp(SNAPSHOT_HEIGHT_DP)
+        .layout();
+  }
+
+  protected void snapshotView(@NonNull View view) {
+    Screenshot.snap(view).setName(getSnapshotName()).record();
+  }
+}

AndroidSDKTests/screenshots/richInterstitial.png

@@ -0,0 +1,63 @@
+package com.leanplum.messagetemplates;
+
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import android.app.Activity;
+import android.graphics.Color;
+import androidx.test.annotation.UiThreadTest;
+import com.leanplum.ActionContext;
+import com.leanplum.Leanplum;
+import com.leanplum.messagetemplates.MessageTemplates.Args;
+import com.leanplum.messagetemplates.MessageTemplates.Values;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+
+public class LPCenterPopupMessageSnapshotTest extends BaseSnapshotTest {
+
+  @Override
+  public String getSnapshotName() {
+    return "centerPopup";
+  }
+
+  @Test
+  @UiThreadTest
+  public void testCenterPopup() {
+    Activity mainActivity = getMainActivity();
+    Leanplum.setApplicationContext(mainActivity);
+
+    String appName = getApplicationName();
+
+    Map<String, Object> args = new HashMap<>();
+    args.put(Args.TITLE_TEXT, appName);
+    args.put(Args.TITLE_COLOR, Color.RED);
+    args.put(Args.MESSAGE_TEXT, Values.POPUP_MESSAGE);
+    args.put(Args.MESSAGE_COLOR, Color.BLACK);
+    args.put(Args.BACKGROUND_COLOR, Color.WHITE);
+    args.put(Args.ACCEPT_BUTTON_TEXT, Values.OK_TEXT);
+    args.put(Args.ACCEPT_BUTTON_BACKGROUND_COLOR, Color.WHITE);
+    args.put(Args.ACCEPT_BUTTON_TEXT_COLOR, Color.BLACK);
+    args.put(Args.LAYOUT_WIDTH, Values.CENTER_POPUP_WIDTH);
+    args.put(Args.LAYOUT_HEIGHT, Values.CENTER_POPUP_HEIGHT);
+    ActionContext realContext = new ActionContext(getSnapshotName(), args, null);
+
+    ActionContext mockedContext = spy(realContext);
+    when(mockedContext.stringNamed(Args.TITLE_TEXT)).thenReturn(appName);
+    when(mockedContext.numberNamed(Args.TITLE_COLOR)).thenReturn(Color.RED);
+    when(mockedContext.stringNamed(Args.MESSAGE_TEXT)).thenReturn(Values.POPUP_MESSAGE);
+    when(mockedContext.numberNamed(Args.MESSAGE_COLOR)).thenReturn(Color.BLACK);
+    when(mockedContext.numberNamed(Args.BACKGROUND_COLOR)).thenReturn(Color.WHITE);
+    when(mockedContext.stringNamed(Args.ACCEPT_BUTTON_TEXT)).thenReturn(Values.OK_TEXT);
+    when(mockedContext.numberNamed(Args.ACCEPT_BUTTON_BACKGROUND_COLOR)).thenReturn(Color.WHITE);
+    when(mockedContext.numberNamed(Args.ACCEPT_BUTTON_TEXT_COLOR)).thenReturn(Color.BLACK);
+    when(mockedContext.numberNamed(Args.LAYOUT_WIDTH)).thenReturn(Values.CENTER_POPUP_WIDTH);
+    when(mockedContext.numberNamed(Args.LAYOUT_HEIGHT)).thenReturn(Values.CENTER_POPUP_HEIGHT);
+
+    CenterPopupOptions options = new CenterPopupOptions(mockedContext);
+    CenterPopup centerpopup = new CenterPopup(mainActivity, options);
+
+    setupView(centerpopup.dialogView);
+    snapshotView(centerpopup.dialogView);
+  }
+}