chore: record failures for e2e tests (#2241)

* chore: record failures for e2e tests

* make test fail

* test better killing of record process

* try this...

* test

* make test fail

* Add recording PID assignment for iOS platform

* cleanup

Author: janicduplessis

bitrise.yml

@@ -465,7 +465,7 @@ workflows:
           inputs:
             - content: |
                 export PATH="${HOME}/.maestro/bin:$PATH"
-                yarn test:e2e:android
+                yarn test:e2e:android --record-on-failure
       - deploy-to-bitrise-io@2:
           is_always_run: true
           inputs:
@@ -562,7 +562,7 @@ workflows:
           inputs:
             - content: |
                 export PATH="${HOME}/.maestro/bin:$PATH"
-                yarn test:e2e:ios
+                yarn test:e2e:ios --record-on-failure
       - deploy-to-bitrise-io@2:
           is_always_run: true
           inputs:

scripts/run-maestro-tests

@@ -1,8 +1,82 @@
 #!/bin/bash
 
-trap 'exit' INT
+set -uo pipefail
 
-PLATFORM=${1:-}
+PLATFORM=""
+RECORD_ON_FAILURE=false
+IS_RECORDING=false
+NEW_ARCH_ENABLED=${NEW_ARCH_ENABLED:-false}
+
+stop_recording() {
+  local recording_dir=$1
+
+  if [ "$IS_RECORDING" = true ]; then
+    echo "Stopping recording..."
+
+    if [ "$PLATFORM" = "android" ]; then
+      adb shell "kill -2 \$(cat /data/local/tmp/recording_pid.txt)" 2>/dev/null || true
+      sleep 2
+      adb pull /data/local/tmp/recording.mp4 "$recording_dir/recording.mp4" 2>/dev/null || true
+      adb shell "rm -f /data/local/tmp/recording.mp4 /data/local/tmp/recording_pid.txt" 2>/dev/null || true
+
+    elif [ "$PLATFORM" = "ios" ]; then
+      killall -SIGINT simctl || true
+    fi
+
+    IS_RECORDING=false
+    echo "Recording saved to $recording_dir/recording.mp4"
+  fi
+}
+
+start_recording() {
+  local recording_dir=$1
+
+  echo "Starting screen recording..."
+  mkdir -p "$recording_dir"
+
+  if [ "$PLATFORM" = "android" ]; then
+    adb shell "screenrecord --bugreport /data/local/tmp/recording.mp4 & echo \$! > /data/local/tmp/recording_pid.txt" &
+    IS_RECORDING=true
+  elif [ "$PLATFORM" = "ios" ]; then
+    xcrun simctl io booted recordVideo --codec=h264 -f "$recording_dir/recording.mp4" &
+    IS_RECORDING=true
+  fi
+}
+
+cleanup() {
+  if [ "$IS_RECORDING" = true ]; then
+    echo "Interrupt cleanup - stopping recording..."
+    if [ "$PLATFORM" = "android" ]; then
+      adb shell "kill -2 \$(cat /data/local/tmp/recording_pid.txt)" 2>/dev/null || true
+    elif [ "$PLATFORM" = "ios" ]; then
+      killall -SIGINT simctl || true
+    fi
+    IS_RECORDING=false
+  fi
+}
+
+handle_interrupt() {
+  cleanup
+  exit 130
+}
+
+trap cleanup EXIT
+trap handle_interrupt INT TERM
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --record-on-failure)
+      RECORD_ON_FAILURE=true
+      ;;
+    *)
+      if [ -z "$PLATFORM" ]; then
+        PLATFORM="$1"
+      fi
+      ;;
+  esac
+  shift
+done
 
 # Validate passed platform
 case $PLATFORM in
@@ -32,6 +106,8 @@ mkdir -p e2e-artifacts
 
 # Optional sharding configuration via environment variables
 # Set SHARD_COUNT to the total number of shards and SHARD_INDEX to the index for this job (0-based)
+SHARD_COUNT=${SHARD_COUNT:-}
+SHARD_INDEX=${SHARD_INDEX:-}
 if { [ -n "$SHARD_COUNT" ] || [ -n "$SHARD_INDEX" ]; } && { [ -z "$SHARD_COUNT" ] || [ -z "$SHARD_INDEX" ]; }; then
   echo "Error! Both SHARD_COUNT and SHARD_INDEX must be set to enable sharding."
   exit 1
@@ -49,9 +125,9 @@ if [ -n "$SHARD_COUNT" ]; then
   echo "Sharding enabled: SHARD_INDEX=$SHARD_INDEX SHARD_COUNT=$SHARD_COUNT"
 fi
 
-# Retry timing configuration (in seconds)
-RETRY_SLEEP_FAST=${RETRY_SLEEP_FAST:-30}
-RETRY_SLEEP_SLOW=${RETRY_SLEEP_SLOW:-120}
+# Retry configuration
+MAX_ATTEMPTS=3
+RETRY_DELAYS=(30 120) # Delay before retry 1, retry 2, etc.
 
 idx=0
 for file in $allTestFiles
@@ -68,38 +144,52 @@ do
     fi
   fi
 
-  if ! eval "$testCmd --debug-output e2e-artifacts/$testName";
-  then
-    echo "Test ${file} failed. Retrying in ${RETRY_SLEEP_FAST}s..."
-    sleep "$RETRY_SLEEP_FAST"
-    if ! eval "$testCmd --debug-output e2e-artifacts/$testName-retry-1";
-    then
-      echo "Test ${file} failed again. Retrying for the last time in ${RETRY_SLEEP_SLOW}s..."
-      sleep "$RETRY_SLEEP_SLOW"
-      if ! eval "$testCmd --debug-output e2e-artifacts/$testName-retry-2";
-      then
-        failedTests+=("$file")
-        # Check if this is a Financial Connections test
-        if [[ "$testName" == *"financial-connections"* ]]; then
-          failedFinancialConnectionsTests+=("$file")
-        fi
+  success=false
+  for attempt in $(seq 1 $MAX_ATTEMPTS); do
+    # Determine artifact directory name
+    if [ "$attempt" -eq 1 ]; then
+      artifactDir="e2e-artifacts/$testName"
+    else
+      artifactDir="e2e-artifacts/$testName-retry-$((attempt - 1))"
+    fi
+
+    # Start recording on retry attempts (after first failure)
+    if [ "$attempt" -gt 1 ] && [ "$RECORD_ON_FAILURE" = "true" ]; then
+      start_recording "$artifactDir"
+    fi
+
+    if eval "$testCmd --debug-output $artifactDir"; then
+      # Stop recording if it was running
+      if [ "$attempt" -gt 1 ] && [ "$RECORD_ON_FAILURE" = "true" ]; then
+        stop_recording "$artifactDir"
       fi
+      success=true
+      break
+    fi
+
+    # Stop recording if it was running
+    if [ "$attempt" -gt 1 ] && [ "$RECORD_ON_FAILURE" = "true" ]; then
+      stop_recording "$artifactDir"
+    fi
+
+    # If not the last attempt, wait before retrying
+    if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then
+      delay=${RETRY_DELAYS[$((attempt - 1))]}
+      echo "Test ${file} failed. Retrying in ${delay}s..."
+      sleep "$delay"
+    fi
+  done
+
+  if [ "$success" = false ]; then
+    failedTests+=("$file")
+    if [[ "$testName" == *"financial-connections"* ]]; then
+      failedFinancialConnectionsTests+=("$file")
     fi
   fi
+
   idx=$((idx + 1))
 done
 
-# Output information for GitHub Actions
-if [ ${#failedFinancialConnectionsTests[@]} -gt 0 ]; then
-    echo "FINANCIAL_CONNECTIONS_TESTS_FAILED=true" >> $GITHUB_OUTPUT
-    echo "FAILED_FC_TESTS=${failedFinancialConnectionsTests[*]}" >> $GITHUB_OUTPUT
-    echo "FINANCIAL_CONNECTIONS_TESTS_FAILED=true" > e2e-artifacts/fincon.env
-    echo "FAILED_FC_TESTS=${failedFinancialConnectionsTests[*]}" >> e2e-artifacts/fincon.env
-else
-    echo "FINANCIAL_CONNECTIONS_TESTS_FAILED=false" >> $GITHUB_OUTPUT
-    echo "FINANCIAL_CONNECTIONS_TESTS_FAILED=false" > e2e-artifacts/fincon.env
-fi
-
 if [ ${#failedTests[@]} -eq 0 ]; then
     exit 0
 else