Kasong/automation add retry (#2504)

* Add back retry on MSAL automation

* Fix typo

* Update typo

* quite MSAL automation logs

* Update ymal for retry

* publish crash logs if any

* publish crash logs if any

* trye to fix all failures

* Add sleep

---------

Co-authored-by: Kai Song <[email protected]>

Author: kaisong1990

MSAL/test/automation/tests/MSALBaseUITest.m

@@ -85,13 +85,20 @@ - (void)assertAccessTokenExpired
 - (void)assertAuthUIAppearsUsingEmbeddedWebView:(BOOL)useEmbedded
 {
     XCUIElement *webElement = self.testApp.buttons[@"URL"];
-
+    
     if (useEmbedded)
     {
         webElement = self.testApp.buttons[@"Cancel"];
     }
+    else
+    {
+        sleep(10);
+        XCUIElement *registerButton = self.testApp.buttons[@"Allow"];
+        XCUIElement *result = [self waitForEitherElements:registerButton and:webElement];
+        [result msidTap];
+    }
 
-    BOOL result = [webElement waitForExistenceWithTimeout:5.0];
+    BOOL result = [webElement waitForExistenceWithTimeout:15.0];
 
     XCTAssertTrue(result);
 }

azure_pipelines/automation.yml

@@ -41,6 +41,12 @@ jobs:
     displayName: 'Check System Information'
     
   - template: templates/tests-with-conf-file.yml
+    parameters:
+      schema: 'MSAL Test Automation (iOS)'
+      full_path: 'build/Build/Products/MSAL Test Automation (iOS)_iphonesimulator18.1-x86_64.xctestrun'
+      destination: 'platform=iOS Simulator,name=iPhone 16,OS=18.1'
+      sdk: 'iphonesimulator'
+      retry_tests: 'true'
 
   - task: UsePythonVersion@0
     condition: failed()
@@ -97,7 +103,7 @@ jobs:
       full_path: 'build/Build/Products/MSAL iOS Native Auth E2E Tests_MSAL iOS Native Auth E2E Tests_iphonesimulator18.1-x86_64.xctestrun'
       destination: 'platform=iOS Simulator,name=iPhone 16,OS=18.1'
       sdk: 'iphonesimulator'
-      retry_tests: false
+      retry_tests: 'false'
 
 - job: e2e_test_native_auth_mac
   displayName: 'Run MSAL E2E tests for macOS native auth'
@@ -113,7 +119,7 @@ jobs:
       full_path: 'build/Build/Products/MSAL Mac Native Auth E2E Tests_MSAL Mac Native Auth E2E Tests_macosx15.1-x86_64.xctestrun'
       destination: 'platform=macOS'
       sdk: 'macosx'
-      retry_tests: false
+      retry_tests: 'false'
 
 - job: cocoapods_lib_lint
   displayName: Run Cocoapods lib lint

azure_pipelines/templates/tests-with-conf-file.yml

@@ -1,10 +1,3 @@
-parameters:
-  schema: 'MSAL Test Automation (iOS)'
-  full_path: 'build/Build/Products/MSAL Test Automation (iOS)_iphonesimulator18.1-x86_64.xctestrun'
-  destination: 'platform=iOS Simulator,name=iPhone 16,OS=18.1'
-  sdk: 'iphonesimulator'
-  retry_tests: true
-
 steps:
   - checkout: self
     clean: true
@@ -56,29 +49,33 @@ steps:
           | tee xcodebuild.log \
           | xcpretty -c
 
-  - task: Bash@3
-    displayName: Run automations
-    inputs:
-      targetType: 'inline'
-      script: |
-        ls build/Build/Products/
-        #Use retry-tests-on-failure only if tests don't already use "Maximum Test Repetitions"
-        if ['${{ parameters.retry_tests }}' == true]; then
-            xcodebuild test-without-building \
-                -xctestrun '${{ parameters.full_path }}' \
-                -destination '${{ parameters.destination }}' \
-                -retry-tests-on-failure \
-                -parallel-testing-enabled NO \
-                -resultBundlePath '$(Agent.BuildDirectory)/s/test_output/report.xcresult'
-
-        else
-            xcodebuild test-without-building \
-                -xctestrun '${{ parameters.full_path }}' \
-                -destination '${{ parameters.destination }}' \
-                -parallel-testing-enabled NO \
-                -resultBundlePath '$(Agent.BuildDirectory)/s/test_output/report.xcresult'
-
-        fi 
+  - ${{ if eq(parameters.retry_tests, 'true') }}:    
+    - task: Bash@3
+      displayName: Run automations
+      inputs:
+        targetType: 'inline'
+        script: |
+          ls build/Build/Products/
+          xcodebuild test-without-building \
+              -xctestrun '${{ parameters.full_path }}' \
+              -destination '${{ parameters.destination }}' \
+              -retry-tests-on-failure \
+              -test-iterations 3 \
+              -parallel-testing-enabled NO \
+              -quiet \
+              -resultBundlePath '$(Agent.BuildDirectory)/s/test_output/report.xcresult'
+  - ${{ if eq(parameters.retry_tests, 'false') }}:    
+    - task: Bash@3
+      displayName: Run automations
+      inputs:
+        targetType: 'inline'
+        script: |
+          ls build/Build/Products/
+          xcodebuild test-without-building \
+              -xctestrun '${{ parameters.full_path }}' \
+              -destination '${{ parameters.destination }}' \
+              -parallel-testing-enabled NO \
+              -resultBundlePath '$(Agent.BuildDirectory)/s/test_output/report.xcresult'
 
   # https://learn.microsoft.com/en-us/azure/devops/pipelines/artifacts/pipeline-artifacts?view=azure-devops&tabs=yaml#q-can-i-delete-pipeline-artifacts-when-re-running-failed-jobs
   - task: PublishPipelineArtifact@1
@@ -87,4 +84,11 @@ steps:
       targetPath: '$(Agent.BuildDirectory)/s/test_output/'
       artifactName: 'TestOutputs Attempt - $(System.StageAttempt) - ${{ parameters.schema }}'
       publishLocation: 'pipeline'
+  - task: PublishPipelineArtifact@1
+    condition: failed()
+    displayName: "Publish crash report if any"
+    inputs:
+      targetPath: "/Users/runner/Library/Logs/DiagnosticReports"
+      artifactName: "crash folder"
+      publishLocation: "pipeline"