diff --git a/.github/DangerFiles/Gemfile.lock b/.github/DangerFiles/Gemfile.lock index b4d502931f..78a979f5e5 100644 --- a/.github/DangerFiles/Gemfile.lock +++ b/.github/DangerFiles/Gemfile.lock @@ -1,79 +1,107 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + activesupport (8.1.1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) + addressable (2.8.8) + public_suffix (>= 2.0.2, < 8.0) ansi (1.5.0) - ast (2.4.2) - base64 (0.2.0) + ast (2.4.3) + base64 (0.3.0) + bigdecimal (3.3.1) claide (1.1.0) claide-plugins (0.9.2) cork nap open4 (~> 1.3) colored2 (3.1.2) + concurrent-ruby (1.3.5) + connection_pool (2.5.5) cork (0.3.0) colored2 (~> 3.1) - danger (9.5.1) + danger (9.5.3) base64 (~> 0.2) claide (~> 1.0) claide-plugins (>= 0.9.2) - colored2 (~> 3.1) + colored2 (>= 3.1, < 5) cork (~> 0.1) faraday (>= 0.9.0, < 3.0) faraday-http-cache (~> 2.0) - git (~> 1.13) - kramdown (~> 2.3) + git (>= 1.13, < 3.0) + kramdown (>= 2.5.1, < 3.0) kramdown-parser-gfm (~> 1.0) octokit (>= 4.0) pstore (~> 0.1) - terminal-table (>= 1, < 4) + terminal-table (>= 1, < 5) danger-android_lint (0.0.12) danger-plugin-api (~> 1.0) oga danger-plugin-api (1.0.0) danger (> 2.0) - faraday (2.12.2) + drb (2.2.3) + faraday (2.14.0) faraday-net_http (>= 2.0, < 3.5) json logger faraday-http-cache (2.5.1) faraday (>= 0.8) - faraday-net_http (3.4.0) - net-http (>= 0.5.0) - git (1.19.1) + faraday-net_http (3.4.2) + net-http (~> 0.5) + git (2.3.3) + activesupport (>= 5.0) addressable (~> 2.8) + process_executer (~> 1.1) rchardet (~> 1.8) - json (2.9.1) + i18n (1.14.7) + concurrent-ruby (~> 1.0) + json (2.16.0) kramdown (2.5.1) rexml (>= 3.3.9) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - logger (1.6.5) + logger (1.7.0) + minitest (5.26.2) nap (1.1.0) - net-http (0.6.0) - uri - octokit (9.2.0) + net-http (0.8.0) + uri (>= 0.11.1) + octokit (10.0.0) faraday (>= 1, < 3) sawyer (~> 0.9) oga (3.4) ast ruby-ll (~> 2.1) open4 (1.3.4) - pstore (0.1.4) - public_suffix (6.0.1) - rchardet (1.9.0) - rexml (3.4.0) - ruby-ll (2.1.3) + process_executer (1.3.0) + pstore (0.2.0) + public_suffix (7.0.0) + rchardet (1.10.0) + rexml (3.4.4) + ruby-ll (2.1.4) ansi ast - sawyer (0.9.2) + sawyer (0.9.3) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) - terminal-table (3.0.2) - unicode-display_width (>= 1.1.1, < 3) - unicode-display_width (2.6.0) - uri (1.0.3) + securerandom (0.4.1) + terminal-table (4.0.0) + unicode-display_width (>= 1.1.1, < 4) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) + uri (1.1.1) PLATFORMS arm64-darwin-23 @@ -84,4 +112,4 @@ DEPENDENCIES danger-android_lint BUNDLED WITH - 2.5.22 + 2.5.16 diff --git a/.github/test-shards/MobileSync.json b/.github/test-shards/MobileSync.json new file mode 100644 index 0000000000..a0c95f91c2 --- /dev/null +++ b/.github/test-shards/MobileSync.json @@ -0,0 +1,39 @@ +{ + "description": "Test shards for MobileSync library - all network tests in one shard to prevent server conflicts", + "shards": [ + { + "name": "network", + "comment": "All network tests in single shard - run sequentially to avoid server conflicts", + "targets": [ + "class com.salesforce.androidsdk.mobilesync.manager.SyncManagerTest", + "class com.salesforce.androidsdk.mobilesync.manager.SyncManagerSuspendTest", + "class com.salesforce.androidsdk.mobilesync.manager.MetadataSyncManagerTest", + "class com.salesforce.androidsdk.mobilesync.manager.LayoutSyncManagerTest", + "class com.salesforce.androidsdk.mobilesync.target.SyncUpTargetTest", + "class com.salesforce.androidsdk.mobilesync.target.CollectionSyncUpTargetTest", + "class com.salesforce.androidsdk.mobilesync.target.SoqlSyncDownTargetTest", + "class com.salesforce.androidsdk.mobilesync.target.RefreshSyncDownTargetTest", + "class com.salesforce.androidsdk.mobilesync.target.BriefcaseSyncDownTargetTest", + "class com.salesforce.androidsdk.mobilesync.target.ParentChildrenSyncTest", + "class com.salesforce.androidsdk.mobilesync.target.ParentChildrenOtherSyncTest" + ] + }, + { + "name": "remaining", + "comment": "All remaining tests - run in parallel, no server conflicts", + "targets": [ + "notClass com.salesforce.androidsdk.mobilesync.manager.SyncManagerTest", + "notClass com.salesforce.androidsdk.mobilesync.manager.SyncManagerSuspendTest", + "notClass com.salesforce.androidsdk.mobilesync.manager.MetadataSyncManagerTest", + "notClass com.salesforce.androidsdk.mobilesync.manager.LayoutSyncManagerTest", + "notClass com.salesforce.androidsdk.mobilesync.target.SyncUpTargetTest", + "notClass com.salesforce.androidsdk.mobilesync.target.CollectionSyncUpTargetTest", + "notClass com.salesforce.androidsdk.mobilesync.target.SoqlSyncDownTargetTest", + "notClass com.salesforce.androidsdk.mobilesync.target.RefreshSyncDownTargetTest", + "notClass com.salesforce.androidsdk.mobilesync.target.BriefcaseSyncDownTargetTest", + "notClass com.salesforce.androidsdk.mobilesync.target.ParentChildrenSyncTest", + "notClass com.salesforce.androidsdk.mobilesync.target.ParentChildrenOtherSyncTest" + ] + } + ] +} diff --git a/.github/test-shards/README.md b/.github/test-shards/README.md new file mode 100644 index 0000000000..2899ac1c6c --- /dev/null +++ b/.github/test-shards/README.md @@ -0,0 +1,20 @@ +# Test Shards + +This directory contains JSON configuration files that define test shards for individual libraries. The overarching goal of sharding is to reduce the time it takes to run tests for PR feedback. Shards run in parallel, which comes with benefits, new potential pitfalls and a small amount of potential maintenance overhead. + +## Benefits + +- Because shards run on seperate devices, this should reduce the unnecessary failures caused by environment pollution. If a test fails to cleanup after itself, it will not affect other shards. + - Please continue to write tests that cleanup after themselves. I do not want to see this become a crutch for flaky tests. +- Faster PR _and_ Nightly runs. + +## Pitfalls + +- Because shards run in parallel, they need to be grouped intelligently. Many of our "unit tests" are actually integration tests. Sharding might save us from database contention, but if tests in seperate shards make API calls to manipulate data in the same org simultaneously this will cause failures. +- It is now possible for tests to run more than once or be skipped if we are not careful. + +## Maintenance + +Each confiruration file defines targets using the `class` keyword. To ensure all tests are run, each config has a "remaining" shard that **only** uses the `notClass` keyword. New test classes will automatically be included in the "remaining" shard. + +However, it is very important that classes added to shards are also added to the "remaining" shard to prevent them from running more than once. Likewise, classes that are removed from shards need to also be removed from the "remaining" shard so they are not skipped. CI will validate this. diff --git a/.github/test-shards/SalesforceHybrid.json b/.github/test-shards/SalesforceHybrid.json new file mode 100644 index 0000000000..05a1903b39 --- /dev/null +++ b/.github/test-shards/SalesforceHybrid.json @@ -0,0 +1,21 @@ +{ + "description": "Test shards for SalesforceHybrid library - isolates network JS bridge tests", + "shards": [ + { + "name": "network-js", + "comment": "JavaScript bridge tests that make server calls - isolated", + "targets": [ + "class com.salesforce.androidsdk.phonegap.ForceJSTest", + "class com.salesforce.androidsdk.phonegap.MobileSyncJSTest" + ] + }, + { + "name": "remaining", + "comment": "All remaining tests - uses notClass to exclude network tests", + "targets": [ + "notClass com.salesforce.androidsdk.phonegap.ForceJSTest", + "notClass com.salesforce.androidsdk.phonegap.MobileSyncJSTest" + ] + } + ] +} diff --git a/.github/test-shards/SalesforceReact.json b/.github/test-shards/SalesforceReact.json new file mode 100644 index 0000000000..045c87605a --- /dev/null +++ b/.github/test-shards/SalesforceReact.json @@ -0,0 +1,23 @@ +{ + "description": "Test shards for SalesforceReact library - isolates network tests to prevent server contention", + "shards": [ + { + "name": "network", + "comment": "OAuth tests that make server calls - isolated", + "targets": [ + "class com.salesforce.androidsdk.reactnative.ReactOAuthTest", + "class com.salesforce.androidsdk.reactnative.ReactNetTest", + "class com.salesforce.androidsdk.reactnative.ReactMobileSyncTest" + ] + }, + { + "name": "remaining", + "comment": "All remaining tests - uses notClass to exclude network tests", + "targets": [ + "notClass com.salesforce.androidsdk.reactnative.ReactOAuthTest", + "notClass com.salesforce.androidsdk.reactnative.ReactNetTest", + "notClass com.salesforce.androidsdk.reactnative.ReactMobileSyncTest" + ] + } + ] +} diff --git a/.github/test-shards/SalesforceSDK.json b/.github/test-shards/SalesforceSDK.json new file mode 100644 index 0000000000..cdf4e2bd96 --- /dev/null +++ b/.github/test-shards/SalesforceSDK.json @@ -0,0 +1,75 @@ +{ + "description": "Test shards for SalesforceSDK library - isolates network tests to prevent server contention", + "shards": [ + { + "name": "network", + "comment": "REST client tests that make live server calls - isolated", + "targets": [ + "class com.salesforce.androidsdk.rest.RestClientTest", + "class com.salesforce.androidsdk.rest.ClientManagerTest", + "class com.salesforce.androidsdk.auth.OAuth2Test", + "class com.salesforce.androidsdk.auth.HttpAccessTest", + "class com.salesforce.androidsdk.analytics.SalesforceAnalyticsManagerTest" + ] + }, + { + "name": "ui", + "comment": "UI tests and security tests that may require user interaction", + "targets": [ + "class com.salesforce.androidsdk.ui.LoginViewActivityTest", + "class com.salesforce.androidsdk.ui.PickerBottomSheetTest", + "class com.salesforce.androidsdk.ui.LoginActivityTest", + "class com.salesforce.androidsdk.ui.LoginOptionsActivityTest", + "class com.salesforce.androidsdk.ui.PickerBottomSheetActivityTest", + "class com.salesforce.androidsdk.ui.DevInfoActivityTest", + "class com.salesforce.androidsdk.security.ScreenLockManagerTest", + "class com.salesforce.androidsdk.security.BiometricAuthenticationManagerTest" + ] + }, + { + "name": "rest-unit", + "comment": "REST unit tests - no network calls", + "targets": [ + "class com.salesforce.androidsdk.rest.RestRequestTest", + "class com.salesforce.androidsdk.rest.ApiVersionStringsTest", + "class com.salesforce.androidsdk.rest.ClientManagerMockTest", + "class com.salesforce.androidsdk.rest.NotificationsTypesResponseBodyTest", + "class com.salesforce.androidsdk.rest.NotificationsApiExceptionTest", + "class com.salesforce.androidsdk.rest.NotificationsApiErrorResponseBodyTest", + "class com.salesforce.androidsdk.rest.NotificationsActionsResponseBodyTest", + "class com.salesforce.androidsdk.rest.files.RenditionTypeTest", + "class com.salesforce.androidsdk.rest.files.ConnectUriBuilderTest", + "class com.salesforce.androidsdk.rest.files.FileRequestsTest" + ] + }, + { + "name": "remaining", + "comment": "All remaining tests - uses notClass/notPackage to exclude tests already in other shards", + "targets": [ + "notClass com.salesforce.androidsdk.rest.RestClientTest", + "notClass com.salesforce.androidsdk.rest.ClientManagerTest", + "notClass com.salesforce.androidsdk.auth.OAuth2Test", + "notClass com.salesforce.androidsdk.auth.HttpAccessTest", + "notClass com.salesforce.androidsdk.analytics.SalesforceAnalyticsManagerTest", + "notClass com.salesforce.androidsdk.ui.LoginViewActivityTest", + "notClass com.salesforce.androidsdk.ui.PickerBottomSheetTest", + "notClass com.salesforce.androidsdk.ui.LoginActivityTest", + "notClass com.salesforce.androidsdk.ui.LoginOptionsActivityTest", + "notClass com.salesforce.androidsdk.ui.PickerBottomSheetActivityTest", + "notClass com.salesforce.androidsdk.ui.DevInfoActivityTest", + "notClass com.salesforce.androidsdk.security.ScreenLockManagerTest", + "notClass com.salesforce.androidsdk.security.BiometricAuthenticationManagerTest", + "notClass com.salesforce.androidsdk.rest.RestRequestTest", + "notClass com.salesforce.androidsdk.rest.ApiVersionStringsTest", + "notClass com.salesforce.androidsdk.rest.ClientManagerMockTest", + "notClass com.salesforce.androidsdk.rest.NotificationsTypesResponseBodyTest", + "notClass com.salesforce.androidsdk.rest.NotificationsApiExceptionTest", + "notClass com.salesforce.androidsdk.rest.NotificationsApiErrorResponseBodyTest", + "notClass com.salesforce.androidsdk.rest.NotificationsActionsResponseBodyTest", + "notClass com.salesforce.androidsdk.rest.files.RenditionTypeTest", + "notClass com.salesforce.androidsdk.rest.files.ConnectUriBuilderTest", + "notClass com.salesforce.androidsdk.rest.files.FileRequestsTest" + ] + } + ] +} \ No newline at end of file diff --git a/.github/workflows/reusable-workflow.yaml b/.github/workflows/reusable-workflow.yaml index 4161336581..cf31a9e40b 100644 --- a/.github/workflows/reusable-workflow.yaml +++ b/.github/workflows/reusable-workflow.yaml @@ -34,7 +34,6 @@ jobs: if: ${{ inputs.lib == 'SalesforceHybrid' }} run: | npm install -g cordova - cordova telemetry off - name: React Native Dependencies if: ${{ inputs.lib == 'SalesforceReact' }} run: npm install -g typescript @@ -75,6 +74,27 @@ jobs: credentials_json: '${{ secrets.GCLOUD_SERVICE_KEY }}' - uses: 'google-github-actions/setup-gcloud@v2' if: success() || failure() + - name: Validate Shard Config + if: success() || failure() + run: | + SHARD_CONFIG=".github/test-shards/${{ inputs.lib }}.json" + if [ -f "$SHARD_CONFIG" ]; then + # Extract sorted class names from non-remaining shards (strip "class " prefix) + CLASSES=$(jq -r '[.shards[] | select(.name != "remaining") | .targets[] | select(startswith("class ")) | ltrimstr("class ")] | sort | .[]' "$SHARD_CONFIG") + + # Extract sorted notClass names from remaining shard (strip "notClass " prefix) + NOT_CLASSES=$(jq -r '[.shards[] | select(.name == "remaining") | .targets[] | select(startswith("notClass ")) | ltrimstr("notClass ")] | sort | .[]' "$SHARD_CONFIG") + + if [ "$CLASSES" != "$NOT_CLASSES" ]; then + echo "::error::Shard config mismatch in $SHARD_CONFIG - classes in shards must exactly match notClass entries in remaining shard" + echo "Difference:" + diff <(echo "$CLASSES") <(echo "$NOT_CLASSES") || true + exit 1 + fi + echo "✓ Shard config valid for ${{ inputs.lib }} ($(echo "$CLASSES" | wc -l | tr -d ' ') classes)" + else + echo "No shard config for ${{ inputs.lib }}, skipping validation" + fi - name: Run Tests continue-on-error: true if: success() || failure() @@ -92,12 +112,27 @@ jobs: RETRIES=1 fi + # Build test-targets-for-shard arguments from config file + SHARD_CONFIG=".github/test-shards/${{ inputs.lib }}.json" + SHARD_ARGS=() + if [ -f "$SHARD_CONFIG" ]; then + NUM_SHARDS=$(jq '.shards | length' "$SHARD_CONFIG") + for i in $(seq 0 $((NUM_SHARDS - 1))); do + # Join targets with semicolons for this shard + TARGETS=$(jq -r ".shards[$i].targets | join(\";\")" "$SHARD_CONFIG") + SHARD_ARGS+=("--test-targets-for-shard=\"${TARGETS}\"") + done + else + SHARD_ARGS=("") + fi + mkdir firebase_results + gcloud components install beta --quiet for LEVEL in $LEVELS_TO_TEST do GCLOUD_RESULTS_DIR=${{ inputs.lib }}-api-${LEVEL}-build-${{github.run_number}} - gcloud firebase test android run \ + eval gcloud beta firebase test android run \ --project mobile-apps-firebase-test \ --type instrumentation \ --app "native/NativeSampleApps/RestExplorer/build/outputs/apk/debug/RestExplorer-debug.apk" \ @@ -107,7 +142,8 @@ jobs: --directories-to-pull=/sdcard \ --results-dir=${GCLOUD_RESULTS_DIR} \ --results-history-name=${{ inputs.lib }} \ - --timeout=40m --no-auto-google-login --no-record-video --no-performance-metrics \ + --timeout=30m --no-auto-google-login --no-record-video --no-performance-metrics \ + "${SHARD_ARGS[@]}" \ --num-flaky-test-attempts=${RETRIES} || true done - name: Copy Test Results