Split release job to sub tasks. (#9809)

granluo · web-flow · commit 2ea640ab5613 · 2022-05-22T23:37:30.000-07:00
* Split release job to sub tasks.
diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -14,20 +14,29 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
-  buildup_SpecsTesting_repo:
+  specs_checking:
     # Don't run on private repo unless it is a PR.
     if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || github.event_name == 'workflow_dispatch'
+    runs-on: macos-12
     env:
       bot_token_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
-      local_repo: specstesting
       # The SDK repo will be cloned to this dir and podspecs from
       # 'podspec_repo_branch' of this repo will be validated and pushed to the
       # testing repo.
       local_sdk_repo_dir: /tmp/test/firebase-ios-sdk
+      local_repo: specstesting
       podspec_repo_branch: master
-    runs-on: macos-12
+    outputs:
+      matrix: ${{ steps.generate_matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Generate matrix
+      id: generate_matrix
+      run: |
+        cd "${GITHUB_WORKSPACE}/ReleaseTooling"
+        swift run manifest --sdk-repo-url "${GITHUB_WORKSPACE}" --spec-output-file-path ./output.json
+        echo "::set-output name=matrix::{\"include\":$( cat output.json )}"
     - name: Get token
       run: |
          scripts/decrypt_gha_secret.sh scripts/gha-encrypted/oss-bot-access.txt.gpg \
@@ -38,13 +47,59 @@ jobs:
       run: |
          ossbotaccess=`cat oss-bot-access.txt`
          BOT_TOKEN="${ossbotaccess}" test_version="${nightly_version}" sdk_version_config="${GITHUB_WORKSPACE}/scripts/create_spec_repo/RC_firebase_sdk.textproto" local_sdk_repo_dir="${local_sdk_repo_dir}" podspec_repo_branch="${podspec_repo_branch}" scripts/release_testing_setup.sh prerelease_testing
+    - name: Clean spec repo
+      run: |
+         ossbotaccess=`cat oss-bot-access.txt`
+         git clone --quiet https://${ossbotaccess}@github.com/Firebase/SpecsTesting.git "${local_repo}"
+         cd "${local_repo}"
+         # Remove all unhidden dirs, i.e. all podspec dir from the spec repo.
+         rm -Rf -- */
+         git add .
+         # commit without diff will throw an error. `git diff --exit-code` can avoid such error.
+         git diff --staged --exit-code || git commit -m "Empty spec repo."
+         git push
+    - name: Clean Artifacts
+      if: ${{ always() }}
+      run: |
+        rm -rf oss-bot-access.txt
+    - uses: actions/upload-artifact@v3
+      with:
+        name: firebase-ios-sdk
+        path: |
+          ${{ env.local_sdk_repo_dir }}/*.podspec
+          ${{ env.local_sdk_repo_dir }}/*.podspec.json
+  buildup_SpecsTesting_repo:
+    needs: specs_checking
+    # Don't run on private repo unless it is a PR.
+    if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || github.event_name == 'workflow_dispatch'
+    runs-on: macos-12
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.specs_checking.outputs.matrix)}}
+    env:
+      bot_token_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
+      local_repo: specstesting
+      local_sdk_repo_dir: /tmp/test/firebase-ios-sdk
+      targeted_pod: ${{ matrix.podspec }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v3
+      with:
+        name: firebase-ios-sdk
+        path: ${{ env.local_sdk_repo_dir }}
+    - name: Get token
+      run: |
+         scripts/decrypt_gha_secret.sh scripts/gha-encrypted/oss-bot-access.txt.gpg \
+         oss-bot-access.txt "$bot_token_secret"
+         scripts/decrypt_gha_secret.sh scripts/gha-encrypted/bot-access.txt.gpg \
+         bot-access.txt "$bot_token_secret"
     - name: Update SpecsTesting repo
       run: |
         botaccess=`cat bot-access.txt`
         cd scripts/create_spec_repo/
         swift build
         pod repo add --silent "${local_repo}" https://"$botaccess"@github.com/Firebase/SpecsTesting.git
-        BOT_TOKEN="${botaccess}" .build/debug/spec-repo-builder --sdk-repo "${local_sdk_repo_dir}" --local-spec-repo-name "${local_repo}" --sdk-repo-name SpecsTesting --github-account Firebase --pod-sources 'https://${BOT_TOKEN}@github.com/Firebase/SpecsTesting' "https://github.com/firebase/SpecsDev.git" "https://github.com/firebase/SpecsStaging.git" "https://cdn.cocoapods.org/" --exclude-pods "FirebaseFirestoreTestingSupport" "FirebaseAuthTestingSupport" "FirebaseCombineSwift"
+        BOT_TOKEN="${botaccess}" .build/debug/spec-repo-builder --sdk-repo "${local_sdk_repo_dir}" --local-spec-repo-name "${local_repo}" --sdk-repo-name SpecsTesting --github-account Firebase --pod-sources 'https://${BOT_TOKEN}@github.com/Firebase/SpecsTesting' "https://github.com/firebase/SpecsDev.git" "https://github.com/firebase/SpecsStaging.git" "https://github.com/CocoaPods/Specs.git" --include-pods "${targeted_pod}" --keep-repo
     - name: Clean Artifacts
       if: ${{ always() }}
       run: |
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,40 +15,92 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
-  buildup_SpecsTesting_repo:
+  specs_checking:
     # Don't run on private repo unless it is a PR.
     if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    runs-on: macos-12
     env:
       bot_token_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
-      local_repo: specstesting
       # The SDK repo will be cloned to this dir and podspecs from
-      # the latest release branch of this repo will be validated and pushed to
-      # the testing repo.
+      # 'podspec_repo_branch' of this repo will be validated and pushed to the
+      # testing repo.
       local_sdk_repo_dir: /tmp/test/firebase-ios-sdk
+      local_repo: specstesting
       podspec_repo_branch: master
+    outputs:
+      matrix: ${{ steps.generate_matrix.outputs.matrix }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Generate matrix
+      id: generate_matrix
+      run: |
+        cd "${GITHUB_WORKSPACE}/ReleaseTooling"
+        swift run manifest --sdk-repo-url "${GITHUB_WORKSPACE}" --spec-output-file-path ./output.json
+        echo "::set-output name=matrix::{\"include\":$( cat output.json )}"
+    - name: Get token
+      run: |
+         scripts/decrypt_gha_secret.sh scripts/gha-encrypted/bot-access.txt.gpg \
+         bot-access.txt "$bot_token_secret"
+    - name: Update SpecsTesting repo setup
+      run: |
+         botaccess=`cat bot-access.txt`
+         BOT_TOKEN="${botaccess}" test_version="${nightly_version}" \
+         sdk_version_config="${GITHUB_WORKSPACE}/scripts/create_spec_repo/RC_firebase_sdk.textproto" \
+         local_sdk_repo_dir="${local_sdk_repo_dir}" \
+         podspec_repo_branch="${podspec_repo_branch}" \
+         scripts/release_testing_setup.sh prerelease_testing
+    - name: Clean spec repo
+      run: |
+         botaccess=`cat bot-access.txt`
+         git clone --quiet https://"$botaccess"@github.com/FirebasePrivate/SpecsTesting.git "${local_repo}"
+         cd "${local_repo}"
+         # Remove all unhided dirs, i.e. all podspec dir from the spec repo.
+         rm -Rf -- */
+         git add .
+         # commit without diff will throw an error. `git diff --exit-code` can avoid such error.
+         git diff --staged --exit-code || git commit -m "Empty spec repo."
+         git push
+    - name: Clean Artifacts
+      if: ${{ always() }}
+      run: |
+        rm -rf bot-access.txt
+    - uses: actions/upload-artifact@v3
+      with:
+        name: firebase-ios-sdk
+        path: |
+          ${{ env.local_sdk_repo_dir }}/*.podspec
+          ${{ env.local_sdk_repo_dir }}/*.podspec.json
+  buildup_SpecsTesting_repo:
+    needs: specs_checking
+    # Don't run on private repo unless it is a PR.
+    if: (github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule') || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
     runs-on: macos-12
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.specs_checking.outputs.matrix)}}
+    env:
+      bot_token_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
+      local_repo: specstesting
+      local_sdk_repo_dir: /tmp/test/firebase-ios-sdk
+      targeted_pod: ${{ matrix.podspec }}
     steps:
     - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@359bebbc29cbe6c87da6bc9ea3bc930432750108
+    - uses: actions/download-artifact@v3
       with:
-        ruby-version: '2.7'
+        name: firebase-ios-sdk
+        path: ${{ env.local_sdk_repo_dir }}
     - name: Get token
       run: |
-         scripts/decrypt_gha_secret.sh scripts/gha-encrypted/oss-bot-access.txt.gpg \
-         oss-bot-access.txt "$bot_token_secret"
          scripts/decrypt_gha_secret.sh scripts/gha-encrypted/bot-access.txt.gpg \
          bot-access.txt "$bot_token_secret"
-    - name: Update SpecsTesting repo setup
-      run: |
-         ossbotaccess=`cat oss-bot-access.txt`
-         BOT_TOKEN="${ossbotaccess}" scripts/release_testing_setup.sh release_testing
     - name: Update SpecsTesting repo
       run: |
         botaccess=`cat bot-access.txt`
         cd scripts/create_spec_repo/
         swift build
         pod repo add --silent "${local_repo}" https://"$botaccess"@github.com/FirebasePrivate/SpecsTesting.git
-        BOT_TOKEN="${botaccess}"  .build/debug/spec-repo-builder --sdk-repo "${local_sdk_repo_dir}" --local-spec-repo-name "${local_repo}" --pod-sources 'https://${BOT_TOKEN}@github.com/FirebasePrivate/SpecsTesting' "https://github.com/firebase/SpecsStaging.git" "https://cdn.cocoapods.org/" --exclude-pods "FirebaseFirestoreTestingSupport" "FirebaseAuthTestingSupport" "FirebaseCombineSwift"
+        BOT_TOKEN="${botaccess}"  .build/debug/spec-repo-builder --sdk-repo "${local_sdk_repo_dir}" --local-spec-repo-name "${local_repo}" --pod-sources 'https://${BOT_TOKEN}@github.com/FirebasePrivate/SpecsTesting' "https://github.com/firebase/SpecsStaging.git" "https://github.com/CocoaPods/Specs.git" --include-pods "${targeted_pod}" --keep-repo
     - name: Clean Artifacts
       if: ${{ always() }}
       run: |
diff --git a/ReleaseTooling/Sources/ManifestParser/GHAMatrixSpecCollector.swift b/ReleaseTooling/Sources/ManifestParser/GHAMatrixSpecCollector.swift
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import ArgumentParser
+import FirebaseManifest
+import Foundation
+import Utils
+
+/// SDKPodspec is to help generate an array of podspec in json file, e.g.
+/// ``` output.json
+/// [{"podspec":"FirebaseABTesting.podspec"},{"podspec":"FirebaseAnalytics.podspec.json"}]
+/// ```
+struct SDKPodspec: Codable {
+  let podspec: String
+}
+
+struct GHAMatrixSpecCollector {
+  var SDKRepoURL: URL
+  var outputSpecFileURL: URL
+  var excludedSDKs: [String] = []
+
+  func getPodsInManifest(_ manifest: Manifest) -> [String] {
+    var podsList: [String] = []
+    for pod in manifest.pods {
+      podsList.append(pod.name)
+    }
+    return podsList
+  }
+
+  func getAllPodspecs() -> [String] {
+    var output: [String] = []
+    let fileManager = FileManager.default
+    let podsSet = Set(getPodsInManifest(FirebaseManifest.shared))
+    do {
+      let fileURLs = try fileManager.contentsOfDirectory(
+        at: SDKRepoURL,
+        includingPropertiesForKeys: nil
+      )
+      for url in fileURLs {
+        let fileNameComponents = url.lastPathComponent.components(separatedBy: ".")
+        if fileNameComponents.count > 1, fileNameComponents[1] == "podspec" {
+          let specName = fileNameComponents[0]
+          podsSet.contains(specName) ? output
+            .append(specName) : print("\(specName) is not in manifiest")
+        }
+      }
+    } catch {
+      print("Error while enumerating files: \(error.localizedDescription)")
+    }
+    return output
+  }
+
+  func generateMatrixJson(to filePath: URL) throws {
+    let testingSpecs = getAllPodspecs()
+    var sdkPodspecs: [SDKPodspec] = []
+    for spec in testingSpecs {
+      sdkPodspecs.append(SDKPodspec(podspec: spec))
+    }
+    // Trim whitespaces so the GitHub Actions matrix can read.
+    let str = try String(
+      decoding: JSONEncoder().encode(sdkPodspecs),
+      as: UTF8.self
+    )
+    try str.trimmingCharacters(in: .whitespacesAndNewlines).write(
+      to: filePath,
+      atomically: true,
+      encoding: String.Encoding.utf8
+    )
+  }
+}
diff --git a/ReleaseTooling/Sources/ManifestParser/main.swift b/ReleaseTooling/Sources/ManifestParser/main.swift
@@ -20,10 +20,25 @@ import Foundation
 import Utils
 
 struct ManifestParser: ParsableCommand {
+  @Option(help: "The path of the SDK repo.",
+          transform: { str in
+            if NSString(string: str).isAbsolutePath { return URL(fileURLWithPath: str) }
+            let documentDir = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+            return documentDir.appendingPathComponent(str)
+          })
+  var SDKRepoURL: URL
+
   /// Path of a text file for Firebase Pods' names.
-  @Option(help: "Output path of a generated file with all Firebase Pods' names.",
-          transform: URL.init(fileURLWithPath:))
-  var podNameOutputFilePath: URL
+  @Option(help: "An output file with Podspecs",
+          transform: { str in
+            if NSString(string: str).isAbsolutePath { return URL(fileURLWithPath: str) }
+            let documentDir = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+            return documentDir.appendingPathComponent(str)
+          })
+  var specOutputFilePath: URL
+
+  @Option(parsing: .upToNextOption, help: "Podspec files that will not be included.")
+  var excludedSpecs: [String]
 
   func parsePodNames(_ manifest: Manifest) throws {
     var output: [String] = []
@@ -32,17 +47,20 @@ struct ManifestParser: ParsableCommand {
     }
     do {
       try output.joined(separator: ",")
-        .write(to: podNameOutputFilePath, atomically: true,
+        .write(to: specOutputFilePath, atomically: true,
                encoding: String.Encoding.utf8)
-      print("\(output) is written in \n \(podNameOutputFilePath).")
+      print("\(output) is written in \n \(specOutputFilePath).")
     } catch {
       throw error
     }
   }
 
   func run() throws {
-    let manifest = FirebaseManifest.shared
-    try parsePodNames(manifest)
+    let specCollector = GHAMatrixSpecCollector(
+      SDKRepoURL: SDKRepoURL,
+      outputSpecFileURL: specOutputFilePath
+    )
+    try specCollector.generateMatrixJson(to: specOutputFilePath)
   }
 }
 
diff --git a/scripts/create_spec_repo/Package.swift b/scripts/create_spec_repo/Package.swift
@@ -20,6 +20,9 @@ import PackageDescription
 
 let package = Package(
   name: "PodSpecBuilder",
+  platforms: [
+    .macOS(.v10_13),
+  ],
   products: [
     .executable(name: "spec-repo-builder", targets: ["SpecRepoBuilder"]),
   ],
diff --git a/scripts/create_spec_repo/Sources/SpecRepoBuilder/main.swift b/scripts/create_spec_repo/Sources/SpecRepoBuilder/main.swift
