ci: Replace Maestro tests with regular UI Tests (#5398)

* Replace Maestro tests with regular UI Tests

* Use reusable workflow

* Fix used lane

* chore: remove .gitignore from project file references

* Update project format

* Update project file

* Use `PreviewProvider` instead of Preview macro

* Use xcodegen for SwiftUITestSample

* Remove `# Logs`

* Disable debugging on tests

* Set project version in xcconfig

* Set `MARKETING_VERSION` version

* Forgot to commit `SwiftUITestSample.xcconfig`

* Refactor uninstall method to include skipIfNotFound parameter and update test teardown logic for cleaner state management

* Set all xcconfig options

* Enable debug again

* Add print available disk space step

Author: itaybre

.github/workflows/ui-tests-common.yml

@@ -47,6 +47,10 @@ jobs:
         shell: bash
         run: system_profiler SPHardwareDataType
 
+      - name: Print available disk space
+        shell: bash
+        run: df -h
+
       - name: Setup Ruby
         uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:

.github/workflows/ui-tests-critical.yml

@@ -8,6 +8,7 @@ on:
     paths:
       - "Sources/**"
       - ".github/workflows/ui-tests-critical.yml"
+      - ".github/workflows/ui-tests-common.yml"
       - "fastlane/**"
       - "TestSamples/**"
       - "**/*.xctestplan"
@@ -21,35 +22,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-sample:
-    name: Build SwiftUITestSample Sample
-    runs-on: macos-15
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Print hardware info
-        run: system_profiler SPHardwareDataType
-      - run: ./scripts/ci-select-xcode.sh "16.2"
-      - uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-      - run: bundle exec fastlane build_ios_swift_ui_test_sample
-      - uses: actions/upload-artifact@v4
-        with:
-          name: SwiftUITestSample
-          path: TestSamples/SwiftUITestSample/DerivedData/Build/Products/Debug-iphonesimulator/SwiftUITestSample.app
-          # If no files are found, fail the job with an error, because they are required for the next step
-          if-no-files-found: error
-
   run-tests:
-    name: Test iOS on Xcode ${{matrix.platform.xcode}} - ${{matrix.test.name}} - V3 # Up the version with every change to keep track of flaky tests
-    needs: [build-sample]
-    runs-on: ${{matrix.platform.runs-on}}
-    env:
-      APP_ARTIFACT_NAME: "SwiftUITestSample"
-      APP_PATH: "Sample.app"
-      MAESTRO_LOGS_PATH: "MaestroLogs"
-      MAESTRO_DRIVER_STARTUP_TIMEOUT: 1500000 # 25 min, CI can be slow at times
+    name: Test iOS on Xcode ${{matrix.platform.xcode}} - V4 # Up the version with every change to keep track of flaky tests
+    uses: ./.github/workflows/ui-tests-common.yml
+    with:
+      fastlane_command: ui_critical_tests_ios_swiftui
+      build_with_make: true
+      xcode_version: ${{matrix.platform.xcode}}
+      macos_version: ${{matrix.platform.runs-on}}
     strategy:
       fail-fast: false
       matrix:
@@ -65,52 +45,3 @@ jobs:
           # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md#installed-sdks
           - runs-on: macos-15
             xcode: "16.2"
-        test:
-          - name: "Crash"
-            maestro_flows_path: "TestSamples/SwiftUITestSample/Maestro/crash.yaml"
-          - name: "CorruptEnvelope"
-            maestro_flows_path: "TestSamples/SwiftUITestSample/Maestro/corruptEnvelope.yaml"
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Print hardware info
-        run: system_profiler SPHardwareDataType
-
-      - name: Install tooling
-        run: make init-ci-test
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: ${{env.APP_ARTIFACT_NAME}}
-          path: ${{env.APP_PATH}}
-
-      - name: Add Microphone permissions
-        uses: ./.github/actions/add-microphone-permissions
-
-      - run: ./scripts/ci-boot-simulator.sh --xcode ${{matrix.platform.xcode}}
-
-      - name: Install App
-        run: xcrun simctl install booted ${{env.APP_PATH}}
-
-      - name: Run Maestro Flows
-        run: |
-          maestro test ${{matrix.test.maestro_flows_path}} --format junit --debug-output ${{env.MAESTRO_LOGS_PATH}} --flatten-debug-output
-
-      - name: Store Maestro Logs
-        uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: maestro-logs-${{matrix.platform.xcode}}-${{matrix.test.name}}
-          path: ${{env.MAESTRO_LOGS_PATH}}
-
-      - name: Archiving Crash Logs
-        uses: actions/upload-artifact@v4
-        if: ${{ failure() || cancelled() }}
-        with:
-          name: crash-logs-${{matrix.platform.xcode}}-${{matrix.test.name}}
-          path: ~/Library/Logs/DiagnosticReports/**
-
-      - name: Store screenshot
-        uses: ./.github/actions/capture-screenshot
-        if: failure()

.gitignore

@@ -95,9 +95,7 @@ Samples/tvOS-Swift/tvOS-Swift.xcodeproj
 Samples/visionOS-Swift/visionOS-Swift.xcodeproj
 Samples/watchOS-Swift/watchOS-Swift.xcodeproj
 Samples/SentrySampleShared/SentrySampleShared.xcodeproj
-
-# Logs
-MaestroLogs/
+TestSamples/SwiftUITestSample/SwiftUITestSample.xcodeproj
 
 Sentry.xcframework*
 Sentry-Dynamic.xcframework*

Brewfile-ci-test

@@ -17,10 +17,6 @@ init-local:
 .PHONY: init-ci-build
 init-ci-build:
 	brew bundle --file Brewfile-ci-build
-
-.PHONY: init-ci-test
-init-ci-test:
-	brew bundle --file Brewfile-ci-test
 
 # installs the tools needed to run CI deploy tasks locally (note that carthage is preinstalled in github actions)
 .PHONY: init-ci-deploy
@@ -187,4 +183,5 @@ xcode:
 	xcodegen --spec Samples/tvOS-Swift/tvOS-Swift.yml
 	xcodegen --spec Samples/visionOS-Swift/visionOS-Swift.yml
 	xcodegen --spec Samples/watchOS-Swift/watchOS-Swift.yml
+	xcodegen --spec TestSamples/SwiftUITestSample/SwiftUITestSample.yml
 	open Sentry.xcworkspace

Makefile

@@ -12,7 +12,13 @@
     "uiTestingScreenshotsLifetime" : "keepAlways"
   },
   "testTargets" : [
-
+    {
+      "target" : {
+        "containerPath" : "container:SwiftUITestSample.xcodeproj",
+        "identifier" : "CB685CB9DBE9C3FFCE8BD314",
+        "name" : "SwiftUITestSampleUITests"
+      }
+    }
   ],
   "version" : 1
 }

Plans/SwiftUITestSample_Base.xctestplan

@@ -0,0 +1,7 @@
+ONLY_ACTIVE_ARCH_Debug = YES
+ONLY_ACTIVE_ARCH_Test = YES
+ONLY_ACTIVE_ARCH_TestCI = YES
+ONLY_ACTIVE_ARCH_Release = NO
+ONLY_ACTIVE_ARCH = $(ONLY_ACTIVE_ARCH_$(CONFIGURATION))
+
+SDKROOT = auto