Merge pull request #2214 from woocommerce/add/ui-test-ci-setup

rachelmcr · web-flow · commit 33d6f1bc5bfd · 2020-05-08T11:27:11.000+01:00
Testing: Add UI test setup on CI with Fastlane
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -4,6 +4,7 @@ orbs:
   # Using 1.0 of the Orbs means it will use the latest 1.0.x version from https://github.com/wordpress-mobile/circleci-orbs
   ios: wordpress-mobile/ios@1.0
   git: wordpress-mobile/git@1.0
+  slack: circleci/slack@3.4.2
 
 commands:
   fix-circleci:
@@ -32,22 +33,80 @@ jobs:
           root: ./
           paths:
             - DerivedData/Build/Products
+            - vendor/bundle
   
   Unit Tests:
     executor:
       name: ios/default
       xcode-version: "11.2.1"
     steps:
+      - git/shallow-checkout
       - ios/boot-simulator:
           xcode-version: "11.2.1"
           device: iPhone 11
       - attach_workspace:
           at: ./
+      - run:
+          name: Prepare Bundle
+          command: bundle --path vendor/bundle
       - ios/wait-for-simulator
-      - ios/xcodebuild:
-          command: test-without-building
-          arguments: -xctestrun DerivedData/Build/Products/WooCommerce_UnitTests_iphonesimulator13.2-x86_64.xctestrun -destination "platform=iOS Simulator,id=$SIMULATOR_UDID"
-      - ios/save-xcodebuild-artifacts
+      - run:
+          name: Run Unit Tests
+          command: >
+            bundle exec fastlane test_without_building
+            xctestrun:DerivedData/Build/Products/WooCommerce_UnitTests_iphonesimulator13.2-x86_64.xctestrun
+            destination:"platform=iOS Simulator,id=$SIMULATOR_UDID" try_count:3
+      - ios/save-xcodebuild-artifacts:
+          result-bundle-path: build/results
+  UI Tests:
+    parameters:
+      device:
+        type: string
+      post-to-slack:
+        description: Post to Slack when tests fail. SLACK_WEBHOOK ENV variable must be set.
+        type: boolean
+        default: false
+    executor:
+      name: ios/default
+      xcode-version: "11.2.1"
+    steps:
+      - git/shallow-checkout
+      - ios/boot-simulator:
+          xcode-version: "11.2.1"
+          device: << parameters.device >>
+      - attach_workspace:
+          at: ./
+      - run:
+          name: Prepare Bundle
+          command: bundle --path vendor/bundle
+      - run:
+          name: Run mocks
+          command: ./WooCommerce/WooCommerceUITests/Mocks/scripts/start.sh 8282
+          background: true
+      - ios/wait-for-simulator
+      - run:
+          name: Run UI Tests
+          command: >
+            bundle exec fastlane test_without_building
+            xctestrun:DerivedData/Build/Products/WooCommerce_UITests_iphonesimulator13.2-x86_64.xctestrun
+            destination:"platform=iOS Simulator,id=$SIMULATOR_UDID" try_count:3
+      - ios/save-xcodebuild-artifacts:
+          result-bundle-path: build/results
+      - when:
+          condition: << parameters.post-to-slack >>
+          steps:
+            - run:
+                name: Prepare Slack message
+                when: always
+                command: |
+                  # Get the name of the device that is running. Using "<< parameters.device >>" can cause slack formatting errors.
+                  DEVICE_NAME=$(xcrun simctl list -j | jq -r --arg UDID $SIMULATOR_UDID '.devices[] | .[] | select(.udid == "\($UDID)") | .name')
+                  echo "export SLACK_FAILURE_MESSAGE=':red_circle: WooCommerce iOS UI tests failed on ${DEVICE_NAME} in \`${CIRCLE_BRANCH}\` branch by ${CIRCLE_USERNAME}.\n\nPlease reach out in #platform9 if you think this failure is not caused by your changes, so we can investigate.'" >> $BASH_ENV
+            - slack/status:
+                fail_only: true
+                include_job_number_field: false
+                include_project_field: false
+                failure_message: '${SLACK_FAILURE_MESSAGE}'
   Installable Build:
     executor:
       name: ios/default
@@ -105,6 +164,47 @@ workflows:
       - Build Tests
       - Unit Tests:
           requires: [ "Build Tests" ]
+      # Always run UI tests on develop and release branches
+      - UI Tests:
+          name: UI Tests (iPhone 11)
+          device: iPhone 11
+          post-to-slack: true
+          requires: [ "Build Tests" ]
+          filters:
+            branches:
+              only:
+                - develop
+                - /^release.*/
+      - UI Tests:
+          name: UI Tests (iPad Air 3rd generation)
+          device: iPad Air \\(3rd generation\\)
+          post-to-slack: true
+          requires: [ "Build Tests" ]
+          filters:
+            branches:
+              only:
+                - develop
+                - /^release.*/
+  #Optionally run UI tests on PRs
+  Optional Tests:
+    jobs:
+      - Hold:
+          type: approval
+          filters:
+            branches:
+              ignore:
+                - develop
+                - /^release.*/
+      - Build Tests:
+          requires: [ "Hold" ]
+      - UI Tests:
+          name: Optional UI Tests (iPhone 11)
+          device: iPhone 11
+          requires: [ "Build Tests" ]
+      - UI Tests:
+          name: Optional UI Tests (iPad Air 3rd generation)
+          device: iPad Air \\(3rd generation\\)
+          requires: [ "Build Tests" ]
   Installable Build:
     jobs:
       - Hold:
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -73,6 +73,7 @@ GEM
     cocoapods-try (1.1.0)
     colored (1.2)
     colored2 (3.1.2)
+    colorize (0.8.1)
     commander-fastlane (4.4.6)
       highline (~> 1.7.2)
     concurrent-ruby (1.1.5)
@@ -135,6 +136,12 @@ GEM
       xcpretty-travis-formatter (>= 0.0.3)
     fastlane-plugin-appcenter (1.6.0)
     fastlane-plugin-sentry (1.6.0)
+    fastlane-plugin-test_center (3.10.2)
+      colorize
+      json
+      plist
+      xcodeproj
+      xctest_list (>= 1.1.8)
     ffi (1.12.2)
     fourflusher (2.3.1)
     fuzzy_match (2.0.4)
@@ -262,6 +269,7 @@ GEM
       rouge (~> 2.0.7)
     xcpretty-travis-formatter (1.0.0)
       xcpretty (~> 0.2, >= 0.0.7)
+    xctest_list (1.1.8)
 
 PLATFORMS
   ruby
@@ -272,6 +280,7 @@ DEPENDENCIES
   fastlane (~> 2)!
   fastlane-plugin-appcenter (= 1.6.0)
   fastlane-plugin-sentry
+  fastlane-plugin-test_center
   fastlane-plugin-wpmreleasetoolkit!
   rake!
   rmagick (~> 3.2.0)
diff --git a/fastlane/Fastfile b/fastlane/Fastfile
@@ -13,7 +13,10 @@ def get_required_env(key)
   ENV[key]
 end
 
-before_all do
+before_all do |lane|
+  # Skip these checks/steps for test lane (not needed for testing)
+  next if lane == :test_without_building
+
   # Check that the env files exist
   unless is_ci || File.file?(USER_ENV_FILE_PATH)
     UI.user_error!("~/.wcios-env.default not found: Please copy env/user.env-example to #{USER_ENV_FILE_PATH} and fill in the values")
@@ -646,4 +649,33 @@ end
 
   def simulator_version()
     return '13.3'
+  end
+
+########################################################################
+# Test Lanes
+########################################################################
+  #####################################################################################
+  # test_without_building
+  # -----------------------------------------------------------------------------------
+  # This lane runs tests without building the app.
+  # It requires a prebuilt xctestrun file and simulator destination where the tests will be run.
+  # -----------------------------------------------------------------------------------
+  # Usage:
+  # bundle exec fastlane test_without_building [xctestrun:<Path to xctestrun file>] [destination:<Simulator>] [try_count:<Number of times to try tests>]
+  #
+  # Example:
+  # bundle exec fastlane test_without_building xctestrun:WooCommerce_UITests_iphonesimulator13.2-x86_64.xctestrun destination:"platform=iOS Simulator,id=$SIMULATOR_UDID" try_count:3
+  #####################################################################################
+  desc "Run tests without building"
+  lane :test_without_building do | options |
+    multi_scan(
+      workspace: "WooCommerce.xcworkspace",
+      scheme: "WooCommerce",
+      test_without_building: true,
+      xctestrun: "#{options[:xctestrun]}",
+      destination: options[:destination],
+      try_count: options[:try_count],
+      output_directory: "build/results",
+      result_bundle: true
+    )
   end
diff --git a/fastlane/Pluginfile b/fastlane/Pluginfile
@@ -9,3 +9,4 @@ end
 gem 'fastlane-plugin-wpmreleasetoolkit', git: 'https://github.com/wordpress-mobile/release-toolkit', tag: '0.9.2'
 gem 'fastlane-plugin-sentry'
 gem 'fastlane-plugin-appcenter', '1.6.0'
+gem 'fastlane-plugin-test_center'
