Adjust UI, add minimum number of retries for ui tests, Fixes AB#3139992

LabApiUtilities/src/main/com/microsoft/identity/labapi/utilities/constants/LabConstants.java

@@ -29,6 +29,8 @@ public class LabConstants {
     public static final String DEFAULT_LAB_SCOPE = "https://request.msidlab.com/.default";
     public static final String KEYVAULT_SCOPE = "https://vault.azure.net/.default";
     public static final String DEFAULT_LAB_CERT_ALIAS = "LabAuth.MSIDLab.com";
+    public static final String MSID_LAB3 = "https://login.microsoftonline.com/msidlab3.com";
+    public static final String MSID_LAB4 = "https://login.microsoftonline.com/msidlab4.com";
 
     static final class UserType {
         public static final String CLOUD = "cloud";

LabApiUtilities/src/test/com/microsoft/identity/labapi/utilities/client/LabClientTest.java

@@ -30,7 +30,9 @@
 import com.microsoft.identity.labapi.utilities.exception.LabApiException;
 import com.microsoft.identity.labapi.utilities.rules.RetryTestRule;
 
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
@@ -43,24 +45,33 @@ public class LabClientTest {
 
     // Give some time for basic user to finish creation to enable rest of test.
     private final long POST_TEMP_USER_CREATION_WAIT = 15000;
+    private LabClient mLabClient;
 
     @Rule
     public RetryTestRule retryRule = new RetryTestRule(3);
 
-    @Test
-    public void canFetchCloudAccount() {
+    @Before
+    public void setup() {
         final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
                 TestBuildConfig.LAB_CLIENT_SECRET
         );
 
-        final LabClient labClient = new LabClient(authenticationClient);
+        mLabClient = new LabClient(authenticationClient);
+    }
+
+    @After
+    public void cleanup() {
+        mLabClient = null;
+    }
 
+    @Test
+    public void canFetchCloudAccount() {
         final LabQuery query = LabQuery.builder()
                 .userType(UserType.CLOUD)
                 .build();
 
         try {
-            final ILabAccount labAccount = labClient.getLabAccount(query);
+            final ILabAccount labAccount = mLabClient.getLabAccount(query);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -74,18 +85,12 @@ public void canFetchCloudAccount() {
 
     @Test
     public void canFetchMSAAccount() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         final LabQuery query = LabQuery.builder()
                 .userType(UserType.MSA)
                 .build();
 
         try {
-            final ILabAccount labAccount = labClient.getLabAccount(query);
+            final ILabAccount labAccount = mLabClient.getLabAccount(query);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -99,18 +104,12 @@ public void canFetchMSAAccount() {
 
     @Test
     public void canFetchGuestAccount() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         final LabQuery query = LabQuery.builder()
                 .userType(UserType.GUEST)
                 .build();
 
         try {
-            final ILabAccount labAccount = labClient.getLabAccount(query);
+            final ILabAccount labAccount = mLabClient.getLabAccount(query);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -124,18 +123,12 @@ public void canFetchGuestAccount() {
 
     @Test
     public void canFetchFederatedAccount() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         final LabQuery query = LabQuery.builder()
                 .userType(UserType.FEDERATED)
                 .build();
 
         try {
-            final ILabAccount labAccount = labClient.getLabAccount(query);
+            final ILabAccount labAccount = mLabClient.getLabAccount(query);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -149,14 +142,8 @@ public void canFetchFederatedAccount() {
 
     @Test
     public void canCreateBasicTempUser() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         try {
-            final ILabAccount labAccount = labClient.createTempAccount(TempUserType.BASIC);
+            final ILabAccount labAccount = mLabClient.createTempAccount(TempUserType.BASIC);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -170,14 +157,8 @@ public void canCreateBasicTempUser() {
 
     @Test
     public void canCreateMAMCATempUser() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         try {
-            final ILabAccount labAccount = labClient.createTempAccount(TempUserType.MAM_CA);
+            final ILabAccount labAccount = mLabClient.createTempAccount(TempUserType.MAM_CA);
             Assert.assertNotNull(labAccount);
             Assert.assertNotNull(labAccount.getUsername());
             Assert.assertNotNull(labAccount.getPassword());
@@ -190,50 +171,32 @@ public void canCreateMAMCATempUser() {
 
     @Test
     public void canResetPassword() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         try {
-            final ILabAccount labAccount = labClient.createTempAccount(TempUserType.BASIC);
+            final ILabAccount labAccount = mLabClient.createTempAccount(TempUserType.BASIC);
             Thread.sleep(POST_TEMP_USER_CREATION_WAIT);
-            Assert.assertTrue(labClient.resetPassword(labAccount.getUsername(), 2));
+            Assert.assertTrue(mLabClient.resetPassword(labAccount.getUsername(), 2));
         } catch (final LabApiException | InterruptedException e) {
             throw new AssertionError(e);
         }
     }
 
     @Test
     public void canEnablePolicy() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         try {
-            final ILabAccount labAccount = labClient.createTempAccount(TempUserType.BASIC);
+            final ILabAccount labAccount = mLabClient.createTempAccount(TempUserType.BASIC);
             Thread.sleep(POST_TEMP_USER_CREATION_WAIT);
-            Assert.assertTrue(labClient.enablePolicy(labAccount.getUsername(), ProtectionPolicy.MAM_CA));
+            Assert.assertTrue(mLabClient.enablePolicy(labAccount.getUsername(), ProtectionPolicy.MAM_CA));
         } catch (final LabApiException | InterruptedException e) {
             throw new AssertionError(e);
         }
     }
 
     @Test
     public void canDisablePolicy() {
-        final LabApiAuthenticationClient authenticationClient = new LabApiAuthenticationClient(
-                TestBuildConfig.LAB_CLIENT_SECRET
-        );
-
-        final LabClient labClient = new LabClient(authenticationClient);
-
         try {
-            final ILabAccount labAccount = labClient.createTempAccount(TempUserType.MAM_CA);
+            final ILabAccount labAccount = mLabClient.createTempAccount(TempUserType.MAM_CA);
             Thread.sleep(POST_TEMP_USER_CREATION_WAIT);
-            Assert.assertTrue(labClient.disablePolicy(labAccount.getUsername(), ProtectionPolicy.MAM_CA));
+            Assert.assertTrue(mLabClient.disablePolicy(labAccount.getUsername(), ProtectionPolicy.MAM_CA));
         } catch (final LabApiException | InterruptedException e){
             throw new AssertionError(e);
         }

uiautomationutilities/build.gradle

@@ -8,6 +8,17 @@ apply from: '../versioning/version_tasks.gradle'
 project.ext.vstsUsername = System.getenv("ENV_VSTS_MVN_ANDROIDCOMMON_USERNAME") != null ? System.getenv("ENV_VSTS_MVN_ANDROIDCOMMON_USERNAME") : project.findProperty("vstsUsername")
 project.ext.vstsPassword = System.getenv("ENV_VSTS_MVN_ANDROIDCOMMON_ACCESSTOKEN") != null ? System.getenv("ENV_VSTS_MVN_ANDROIDCOMMON_ACCESSTOKEN") : project.findProperty("vstsMavenAccessToken")
 
+def minimumTestAttempts = 3
+
+if (project.hasProperty("testAttempts")) {
+    minimumTestAttempts = testAttempts
+}
+
+def uiElementTimeoutConfig = 25
+
+if (project.hasProperty("uiElementTimeout")) {
+    uiElementTimeoutConfig = uiElementTimeout
+}
 
 android {
 
@@ -108,6 +119,8 @@ android {
         buildConfigField("boolean", "PREFER_PRE_INSTALLED_APKS", "$usePreInstalledApks")
         buildConfigField("boolean", "PRE_INSTALL_LTW", "$preInstallLtw")
         buildConfigField("boolean", "SEND_POWERLIFT_LOGS", "$sendPowerLiftLogs")
+        buildConfigField("int", "MINIMUM_TEST_ATTEMPTS", "$minimumTestAttempts")
+        buildConfigField("int", "UI_ELEMENT_TIMEOUT", "$uiElementTimeoutConfig")
 
         // Specifies a sorted list of flavors that the plugin should try to use from
         // a given dimension. The following tells the plugin that, when encountering

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/broker/BrokerMicrosoftAuthenticator.java

@@ -31,6 +31,7 @@
 import android.os.Build;
 import android.widget.Button;
 import android.widget.EditText;
+import android.widget.ImageView;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
@@ -190,37 +191,35 @@ public String createPowerLiftIncident() {
     }
 
     private String createPowerLiftIncidentInNonSharedMode() {
-        // click the 3 dot menu icon in top right
-        UiAutomatorUtils.handleButtonClick("com.azure.authenticator:id/menu_overflow");
-
         try {
-            // select Help from drop down
-            final UiObject settings = UiAutomatorUtils.obtainUiObjectWithText("Send Feedback");
-            settings.click();
+            // click the 3 dot menu icon in top right
+            final UiObject menu = UiAutomatorUtils.obtainUiObjectWithDescription("More options. Action required.");
+            menu.click();
 
-            final UiObject sendLogs = UiAutomatorUtils.obtainUiObjectWithClassAndDescription(
-                    Button.class,
-                    "Having trouble?Report it"
-            );
+            // select Send Feedback from drop down
+            final UiObject sendFeedback = UiAutomatorUtils.obtainUiObjectWithDescription("Send Feedback");
+            sendFeedback.click();
+
+            final UiObject havingTrouble = UiAutomatorUtils.obtainUiObjectWithText("Having trouble?");
 
-            Assert.assertTrue(sendLogs.exists());
+            Assert.assertTrue(havingTrouble.exists());
 
-            // click the send logs button
-            sendLogs.click();
+            // click the havingTrouble button
+            havingTrouble.click();
 
             UiAutomatorUtils.handleButtonClickForObjectWithText("Select an option");
 
             UiAutomatorUtils.handleButtonClickForObjectWithText("Other");
 
-            final UiObject describeIssueBox = UiAutomatorUtils.obtainUiObjectWithTextAndClassType(
-                    "Please don't include your name, phone number, or other personal information.",
-                    EditText.class
+            final UiObject describeIssueBox = UiAutomatorUtils.obtainUiObjectWithDescription(
+                    "Describe the issue you are facing"
             );
 
             describeIssueBox.setText(INCIDENT_MSG);
 
-            final UiObject sendBtn = UiAutomatorUtils.obtainUiObjectWithDescription("Send feedback");
-            sendBtn.click();
+            // Send incident button also has send feedback description
+            final UiObject sendIncident = UiAutomatorUtils.obtainUiObjectWithDescription("Send feedback");
+            sendIncident.click();
 
             final UiObject postLogSubmissionMsg = UiAutomatorUtils.obtainUiObjectWithResourceId(
                     "android:id/parentPanel"

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/device/settings/GoogleSettings.java

@@ -24,6 +24,8 @@
 
 import static com.microsoft.identity.client.ui.automation.utils.CommonUtils.FIND_UI_ELEMENT_TIMEOUT_LONG;
 import static com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils.handleButtonClick;
+import static com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils.handleButtonClickForObjectWithClass;
+import static com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils.handleButtonClickForObjectWithExactText;
 import static com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils.obtainUiObjectWithExactText;
 
 import androidx.annotation.NonNull;
@@ -356,6 +358,16 @@ public void disableAppThroughSettings(@NonNull final String packageName) {
         handleButtonClick("android:id/button1");
     }
 
+    @Override
+    public void toggleNotificationsThroughSettings(@NonNull final String packageName) {
+        Logger.i(TAG, "Disabling app through settings: " + packageName);
+        launchAppInfoPage(packageName);
+        // Open Notifications page
+        handleButtonClickForObjectWithExactText("Notifications");
+        // Toggle notifications switch
+        handleButtonClickForObjectWithClass("android.widget.Switch");
+    }
+
     @Override
     public void enableAppThroughSettings(@NonNull final String packageName) {
         Logger.i(TAG, "Enabling app through settings");

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/device/settings/ISettings.java

@@ -136,6 +136,12 @@ void addWorkAccount(final ITestBroker expectedBroker,
      */
     public void disableAppThroughSettings(@NonNull final String packageName);
 
+    /**
+     * Toggle app notifications in settings page
+     * @param packageName package name of the app
+     */
+    public void toggleNotificationsThroughSettings(@NonNull final String packageName);
+
     /**
      * Enable an app through the device's settings page instead of shell command.
      * @param packageName name of package to be enabled

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/device/settings/SamsungSettings.java

@@ -222,6 +222,11 @@ public void disableAppThroughSettings(@NonNull final String packageName) {
         throw new UnsupportedOperationException("We do not support disabling apps through Settings Page on samsung devices");
     }
 
+    @Override
+    public void toggleNotificationsThroughSettings(@NonNull String packageName) {
+        throw new UnsupportedOperationException("We do not support disabling notifications through settings page on samsung devices.");
+    }
+
     @Override
     public void enableAppThroughSettings(@NonNull final String packageName) {
         //TODO: implement enabling app through settings

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/rules/RetryTestRule.java

@@ -22,6 +22,7 @@
 //  THE SOFTWARE.
 package com.microsoft.identity.client.ui.automation.rules;
 
+import com.microsoft.identity.client.ui.automation.BuildConfig;
 import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.logging.Logger;
 
@@ -36,6 +37,7 @@
 public class RetryTestRule implements TestRule {
 
     private final static String TAG = RetryTestRule.class.getSimpleName();
+    private final static int MINIMUM_NUMBER_OF_ATTEMPTS = BuildConfig.MINIMUM_TEST_ATTEMPTS;
 
     @Override
     public Statement apply(final Statement base, final Description description) {
@@ -61,6 +63,11 @@ public void evaluate() throws Throwable {
                     numAttempts += retryCount;
                 }
 
+                // If after evaluating annotation, we still have number of attempts less than minimum, increase number of attempts
+                if (numAttempts < MINIMUM_NUMBER_OF_ATTEMPTS) {
+                    numAttempts = MINIMUM_NUMBER_OF_ATTEMPTS;
+                }
+
                 for (int i = 0; i < numAttempts; i++) {
                     try {
                         Logger.i(TAG, "Executing attempt #" + (i + 1) + " of " + numAttempts);

uiautomationutilities/src/main/java/com/microsoft/identity/client/ui/automation/utils/CommonUtils.java

@@ -33,6 +33,7 @@
 import androidx.annotation.Nullable;
 import androidx.test.core.app.ApplicationProvider;
 
+import com.microsoft.identity.client.ui.automation.BuildConfig;
 import com.microsoft.identity.client.ui.automation.broker.BrokerCompanyPortal;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
 import com.microsoft.identity.client.ui.automation.broker.BrokerLTW;
@@ -49,7 +50,7 @@ public class CommonUtils {
 
     private final static String TAG = CommonUtils.class.getSimpleName();
     public final static long FIND_UI_ELEMENT_TIMEOUT_SHORT = TimeUnit.SECONDS.toMillis(10);
-    public final static long FIND_UI_ELEMENT_TIMEOUT = TimeUnit.SECONDS.toMillis(15);
+    public final static long FIND_UI_ELEMENT_TIMEOUT = TimeUnit.SECONDS.toMillis(BuildConfig.UI_ELEMENT_TIMEOUT);
     public final static long FIND_UI_ELEMENT_TIMEOUT_LONG = TimeUnit.SECONDS.toMillis(30);
 
     private final static String SD_CARD = "/sdcard";