Set up multi jobs for spec testing. (#9148)

granluo · web-flow · commit f8e47afc5bfb · 2022-01-04T13:21:45.000-08:00
* Set up multi jobs for spec testing.
diff --git a/.github/workflows/abtesting.yml b/.github/workflows/abtesting.yml
@@ -118,14 +118,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           FirebaseABTesting.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseABTesting.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml
@@ -33,14 +33,3 @@ jobs:
 
 # TODO: Consider pushing GoogleAppMeasurement.podspec.json to SpecsDev to enable similar test
 # for FirebaseAnalytics.podspec.json
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseAnalyticsSwift.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/auth.yml b/.github/workflows/auth.yml
@@ -141,14 +141,3 @@ jobs:
       run: scripts/configure_test_keychain.sh
     - name: PodLibLint Auth Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseAuth.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseAuth.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
@@ -94,14 +94,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint Core Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseCore.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseCore.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/crashlytics.yml b/.github/workflows/crashlytics.yml
@@ -129,14 +129,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint Auth Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseCrashlytics.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseCrashlytics.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml
@@ -150,14 +150,3 @@ jobs:
       run: scripts/setup_bundler.sh
     - name: PodLibLint database Cron
       run: scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseDatabase.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseDatabase.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/dynamiclinks.yml b/.github/workflows/dynamiclinks.yml
@@ -82,14 +82,3 @@ jobs:
     - name: Test swift quickstart
       if: ${{ always() }}
       run: ([ -z $plist_secret ] || scripts/third_party/travis/retry.sh scripts/test_quickstart.sh DynamicLinks true swift)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseDynamicLinks.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/firestore.yml b/.github/workflows/firestore.yml
@@ -289,14 +289,3 @@ jobs:
     - name: Test swift quickstart
       run: ([ -z $plist_secret ] ||
             scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Firestore false)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseFirestore.podspec --skip-tests --allow-warnings --platforms=ios --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/functions.yml b/.github/workflows/functions.yml
@@ -145,17 +145,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           FirebaseFunctions.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        target: [ios, tvos, macos, watchos]
-        podspec: [ 'FirebaseFunctions.podspec', 'FirebaseFunctionsSwift.podspec']
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.podspec }} --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/google-utilities-components.yml b/.github/workflows/google-utilities-components.yml
@@ -64,14 +64,3 @@ jobs:
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb \
           GoogleUtilitiesComponents.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint GoogleUtilitiesComponents.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/inappmessaging.yml b/.github/workflows/inappmessaging.yml
@@ -107,17 +107,3 @@ jobs:
     - name: Test swift quickstart
       run: ([ -z $plist_secret ] ||
             scripts/third_party/travis/retry.sh scripts/test_quickstart.sh InAppMessaging true swift)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        podspec: [FirebaseInAppMessaging.podspec, FirebaseInAppMessagingSwift.podspec]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.podspec}} --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/installations.yml b/.github/workflows/installations.yml
@@ -134,14 +134,3 @@ jobs:
        export FIS_INTEGRATION_TESTS_REQUIRED=${{ steps.secrets.outputs.val }}
        scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseInstallations.podspec \
          --platforms=${{ matrix.target }} ${{ matrix.flags }} \
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseInstallations.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/messaging.yml b/.github/workflows/messaging.yml
@@ -198,14 +198,3 @@ jobs:
       run: scripts/install_prereqs.sh MessagingSample iOS
     - name: Build
       run: ([ -z $plist_secret ] || scripts/build.sh MessagingSample iOS)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseMessaging.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/mlmodeldownloader.yml b/.github/workflows/mlmodeldownloader.yml
@@ -122,14 +122,3 @@ jobs:
       run: scripts/install_prereqs.sh MLModelDownloaderSample iOS
     - name: Build
       run: ([ -z $plist_secret ] || scripts/build.sh MLModelDownloaderSample iOS)
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseMLModelDownloader.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -121,17 +121,6 @@ jobs:
     - name: Setup project and Build Catalyst
       run: scripts/test_catalyst.sh FirebasePerformance build
 
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebasePerformance.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
-
   performance-cron-only:
     # Don't run on private repo.
     if: github.event_name == 'schedule' && github.repository == 'Firebase/firebase-ios-sdk'
diff --git a/.github/workflows/remoteconfig.yml b/.github/workflows/remoteconfig.yml
@@ -158,14 +158,3 @@ jobs:
     - name: PodLibLint RemoteConfig Cron
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseRemoteConfig.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint FirebaseRemoteConfig.podspec --skip-tests --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/.github/workflows/spectesting.yml b/.github/workflows/spectesting.yml
@@ -0,0 +1,62 @@
+name: spectesting
+
+on:
+  pull_request
+
+jobs:
+  specs_checking:
+    # Don't run on private repo unless it is a PR.
+    if: github.repository == 'Firebase/firebase-ios-sdk'
+    runs-on: macos-11
+    outputs:
+      matrix: ${{ steps.check_files.outputs.matrix }}
+      podspecs: ${{ steps.check_files.outputs.podspecs }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: check files
+      id: check_files
+      env:
+        pr_branch: ${{ github.event.pull_request.head.ref }}
+      run: |
+        # The output.json will have podspecs that will be tested.
+        # ``` output.json
+        # [{"podspec":"FirebaseABTesting.podspec"},{"podspec":"FirebaseAnalytics.podspec.json"}]
+        # ```
+        # This array will help generate a matrix for jobs in GHA, taking
+        # advantage of `include`.
+        # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-including-additional-values-into-combinations
+        # The final matrix will be set as an env var in the GHA workflow,
+        # This env var, matrix, will be passed as a matrix in the next job of
+        # the workflow.
+        ./scripts/health_metrics/get_updated_files.sh -p output.json
+        echo "::set-output name=matrix::{\"include\":$( cat output.json )}"
+        # `podspecs` is to help determine if specs_testing job should be run.
+        echo "::set-output name=podspecs::$(cat output.json)"
+  specs_testing:
+    needs: specs_checking
+    if: ${{ needs.specs_checking.outputs.podspecs != '[]' }}
+    runs-on: macos-11
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.specs_checking.outputs.matrix)}}
+    env:
+      PODSPEC: ${{ matrix.podspec }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Init podspecs and source
+      run: |
+        mkdir specTestingLogs
+        cd ReleaseTooling
+        swift run podspecs-tester --git-root "${GITHUB_WORKSPACE}" --podspec ${PODSPEC} --skip-tests --temp-log-dir "${GITHUB_WORKSPACE}/specTestingLogs"
+    - name: Upload Failed Testing Logs
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: specTestingLogs
+        path: specTestingLogs/*.txt
diff --git a/.github/workflows/storage.yml b/.github/workflows/storage.yml
@@ -170,20 +170,3 @@ jobs:
     - name: PodLibLint Storage Cron
       run: |
         scripts/third_party/travis/retry.sh scripts/pod_lib_lint.rb FirebaseStorageSwift.podspec --platforms=${{ matrix.target }} ${{ matrix.flags }}
-
-  podspec-presubmit:
-    # Don't run on private repo unless it is a PR.
-    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.pull_request.merged != true && github.event.action != 'closed'
-    runs-on: macos-11
-    strategy:
-      matrix:
-        spec: [
-          'FirebaseStorage.podspec --test-specs=unit', # The integration tests need more set up.
-          'FirebaseStorageSwift.podspec --skip-tests' # No current unit tests.
-        ]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Bundler
-      run: scripts/setup_bundler.sh
-    - name: Build and test
-      run: scripts/third_party/travis/retry.sh pod spec lint ${{ matrix.spec }} --sources='https://github.com/firebase/SpecsTesting','https://github.com/firebase/SpecsDev.git','https://github.com/firebase/SpecsStaging.git','https://cdn.cocoapods.org/'
diff --git a/ReleaseTooling/Sources/PodspecsTester/InitializeSource.swift b/ReleaseTooling/Sources/PodspecsTester/InitializeSource.swift
@@ -55,20 +55,31 @@ struct InitializeSpecTesting {
                                   podRepoName: String = Constants.localSpecRepoName) {
     let result = Shell.executeCommandFromScript("pod repo remove \(podRepoName)")
     switch result {
-    case let .error(code, output):
+    case let .error(_, output):
       print("\(podRepoName) was not properly removed. \(podRepoName) probably" +
         "does not exist in local.\n \(output)")
-    case let .success(output):
+    case .success:
       print("\(podRepoName) was removed.")
     }
     Shell.executeCommand("pod repo add \(podRepoName) \(repoURL)")
   }
 
   // Add a testing tag to the head of the branch.
   private static func addTestingTag(path sdkRepoPath: URL, manifest: FirebaseManifest.Manifest) {
-    let testingTag = Constants.testingTagPrefix + manifest.version
-    // Add or update the testing tag to the local sdk repo.
-    Shell.executeCommand("git tag -af \(testingTag) -m 'spectesting'", workingDir: sdkRepoPath)
+    // Pods could have different versions, like `8.11.0` and `8.11.0-beta`.
+    // These versions should be part of tags, so a warning from `pod spec lint`
+    // could be avoided.
+    // ```
+    //   The version should be included in the Git tag.
+    // ```
+    // The tag should include `s.version`, e.g.
+    // If "s.version = '8.11.0-beta'", the tag should include 8.11.0-beta to
+    // avoid trigerring the warning.
+    for pod in manifest.pods {
+      let testingTag = Constants.testingTagPrefix + manifest.versionString(pod)
+      // Add or update the testing tag to the local sdk repo.
+      Shell.executeCommand("git tag -af \(testingTag) -m 'spectesting'", workingDir: sdkRepoPath)
+    }
   }
 
   // Update the podspec source.
@@ -85,11 +96,11 @@ struct InitializeSpecTesting {
         // After `sed`:
         //  s.source           = {
         //    :git => '\(path.path)',
-        //    :tag => 'testing-\(manifest.version)',
+        //    :tag => 'testing-\(version)',
         //  }
         Shell.executeCommand(
           "sed -i.bak -e \"s|\\(.*\\:git =>[[:space:]]*\\).*|\\1'\(path.path)',| ; " +
-            "s|\\(.*\\:tag =>[[:space:]]*\\).*|\\1'\(Constants.testingTagPrefix + manifest.version)',|\" \(pod.name).podspec",
+            "s|\\(.*\\:tag =>[[:space:]]*\\).*|\\1'\(Constants.testingTagPrefix + version)',|\" \(pod.name).podspec",
           workingDir: path
         )
       }
diff --git a/ReleaseTooling/Sources/PodspecsTester/main.swift b/ReleaseTooling/Sources/PodspecsTester/main.swift
diff --git a/scripts/health_metrics/code_coverage_file_list.json b/scripts/health_metrics/code_coverage_file_list.json
diff --git a/scripts/health_metrics/generate_code_coverage_report/Sources/UpdatedFilesCollector/main.swift b/scripts/health_metrics/generate_code_coverage_report/Sources/UpdatedFilesCollector/main.swift
diff --git a/scripts/health_metrics/get_updated_files.sh b/scripts/health_metrics/get_updated_files.sh
