Add Code Coverage Report. (#7357)

Create coverage reports for PRs when PRs are updated.

Author: granluo

.github/workflows/test_coverage.yml

@@ -1,15 +1,19 @@
 name: test_coverage
 
-on: [pull_request]
-#run specific jobs when specific files are updated.
-#https://github.community/t/how-to-execute-jobs-if-folder-updated-recursive/117344/5
+on:
+  pull_request:
+    # synchronize will be triggered when a pull request has new commits.
+    # closed will be triggered when a pull request is closed.
+    types: [synchronize, closed]
 
 jobs:
   check:
+    if: github.repository == 'Firebase/firebase-ios-sdk' && github.event.action == 'synchronize'
     name: Check changed files
     outputs:
       database_run_job: ${{ steps.check_files.outputs.database_run_job }}
       functions_run_job: ${{ steps.check_files.outputs.functions_run_job }}
+      base_commit: ${{ steps.check_files.outputs.base_commit }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -21,37 +25,34 @@ jobs:
         env:
           pr_branch: ${{ github.event.pull_request.head.ref }}
         run: ./scripts/code_coverage_report/get_updated_files.sh
+
   pod-lib-lint-database:
     # Don't run on private repo unless it is a PR.
+    if: github.repository == 'Firebase/firebase-ios-sdk' && (needs.check.outputs.database_run_job == 'true' || github.event.pull_request.merged == true)
     needs: check
-    if: github.repository == 'Firebase/firebase-ios-sdk' && needs.check.outputs.database_run_job == 'true'
     runs-on: macOS-latest
-
     strategy:
       matrix:
-        target: [ios, tvos, macos]
+        target: [ios]
     steps:
     - uses: actions/checkout@v2
     - name: Setup Bundler
       run: scripts/setup_bundler.sh
     - name: Build and test
-      env:
-        SDK: database
       run: ./scripts/code_coverage_report/pod_test_code_coverage_report.sh FirebaseDatabase "${{ matrix.target }}"
     - uses: actions/upload-artifact@v2
       with:
-        name: database-codecoverage
+        name: codecoverage
         path: /Users/runner/*.xcresult
 
   pod-lib-lint-functions:
     # Don't run on private repo unless it is a PR.
+    if: github.repository == 'Firebase/firebase-ios-sdk' && (needs.check.outputs.functions_run_job == 'true' || github.event.pull_request.merged == true)
     needs: check
-    if: github.repository == 'Firebase/firebase-ios-sdk' && needs.check.outputs.functions_run_job == 'true'
     runs-on: macOS-latest
-
     strategy:
       matrix:
-        target: [ios, tvos, macos]
+        target: [ios]
     steps:
     - uses: actions/checkout@v2
     - name: Setup Bundler
@@ -60,20 +61,48 @@ jobs:
       run: ./scripts/code_coverage_report/pod_test_code_coverage_report.sh FirebaseFunctions "${{ matrix.target }}"
     - uses: actions/upload-artifact@v2
       with:
-        name: functions-codecoverage
+        name: codecoverage
         path: /Users/runner/*.xcresult
 
   create_report:
-    needs: [pod-lib-lint-functions, pod-lib-lint-database]
+    needs: [check, pod-lib-lint-functions, pod-lib-lint-database]
+    env:
+      metrics_service_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
     if: always()
     runs-on: macOS-latest
     steps:
+      - uses: actions/checkout@v2
+      - name: Access to Metrics Service
+        run: |
+          # Install gcloud sdk
+          curl https://sdk.cloud.google.com > install.sh
+          bash install.sh --disable-prompts
+          echo "${HOME}/google-cloud-sdk/bin/" >> $GITHUB_PATH
+          export PATH="${HOME}/google-cloud-sdk/bin/:${PATH}"
+
+          # Activate the service account for Metrics Service.
+          scripts/decrypt_gha_secret.sh scripts/gha-encrypted/metrics_service_access.json.gpg \
+          metrics-access.json "$metrics_service_secret"
+          gcloud auth activate-service-account --key-file metrics-access.json
       - uses: actions/download-artifact@v2
         id: download
         with:
           path: /Users/runner/test
-      - name: display results
+      - name: Compare Diff and Post a Report
+        if: github.event_name == 'pull_request'
+        env:
+          base_commit: ${{ needs.check.outputs.base_commit }}
+        run: |
+          # Get Head commit of the branch, instead of a merge commit created by actions/checkout.
+          GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
+          if [ -d "${{steps.download.outputs.download-path}}" ]; then
+          cd scripts/code_coverage_report/generate_code_coverage_report
+          swift run CoverageReportGenerator --presubmit "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" --token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" --log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" --pull-request-num ${{github.event.pull_request.number}} --base-commit "$base_commit"
+          fi
+      - name: Update New Coverage Data
+        if: github.event.pull_request.merged == true
         run: |
           if [ -d "${{steps.download.outputs.download-path}}" ]; then
-          find "/Users/runner/test" -print -regex ".*/.*\.xcresult" -exec xcrun xccov view --report {} \;
+          cd scripts/code_coverage_report/generate_code_coverage_report
+          swift run CoverageReportGenerator --merge "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" --token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" --log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" --branch "${GITHUB_REF##*/}"
           fi

.gitignore

@@ -69,6 +69,8 @@ DerivedData
 ReleaseTooling/Packages
 ReleaseTooling/*.xcodeproj
 ReleaseTooling/Package.resolved
+scripts/code_coverage_report/generate_code_coverage_report/Package.resolved
+scripts/code_coverage_report/generate_code_coverage_report/.build
 
 # Mint package manager
 Mint

scripts/code_coverage_report/generate_code_coverage_report/Package.swift

@@ -0,0 +1,45 @@
+// swift-tools-version:5.3
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PackageDescription
+
+let package = Package(
+  name: "CoverageReportGenerator",
+  products: [
+    // Products define the executables and libraries a package produces, and make them visible to other packages.
+    .executable(
+      name: "CoverageReportGenerator",
+      targets: ["CoverageReportGenerator"]
+    ),
+  ],
+  dependencies: [
+    // Dependencies declare other packages that this package depends on.
+    // .package(url: /* package url */, from: "1.0.0"),
+    .package(url: "https://github.com/apple/swift-argument-parser", from: "0.2.0"),
+  ],
+  targets: [
+    // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+    // Targets can depend on other targets in this package, and on products in packages this package depends on.
+    .target(
+      name: "CoverageReportGenerator",
+      dependencies: [
+        .product(name: "ArgumentParser", package: "swift-argument-parser"),
+      ]
+    ),
+  ]
+)

scripts/code_coverage_report/generate_code_coverage_report/README.md

@@ -0,0 +1,60 @@
+# coverage_report_parser
+
+This is a tool to read test coverages of xcresult bundle and generate a json report. This json
+report will be sent to Metrics Service to create a coverage report as a comment in a PR or to update
+the coverage database.
+
+## Usage
+
+This is tool will be used for both pull_request and merge. Common flags are shown below.
+
+```
+swift run CoverageReportGenerator --presubmit "${REPO}" --commit "${GITHUB_SHA}" --token "${TOKEN}" \
+--xcresult-dir "${XCRESULT_DIR}" --log-link "${}" --pull-request-num "${PULL_REQUEST_NUM}" \
+--base-commit "${BASE_COMMIT}" --branch "${BRANCH}"
+```
+Common parameters for both pull_request and merge:
+- `presubmit/merge`: A required flag to know if the request is for pull requests or merge.
+- `REPO`: A required argument for a repo where coverage data belong.
+- `commit`: The current commit sha.
+- `token`: A token to access a service account of Metrics Service
+- `xcresult-dir`: A directory containing all xcresult bundles.
+
+### Create a report in a pull request
+
+In a workflow, this will run for each pull request update. The command below will generate a report
+in a PR. After a workflow of test coverage is done, a new coverage report will be posted on a
+comment of a pull request. If such comment has existed, this comment will be overriden by the latest
+report.
+
+Since the flag is `presubmit` here, the following options are required for a PR request:
+- `log-link`: Log link to unit tests. This is generally a actions/runs/ link in Github Actions.
+- `pull-request-num`: A report will be posted in this pull request.
+- `base-commit`: The commit sha used to compare the diff of the current`commit`.
+
+An example in a Github Actions workflow:
+```
+swift run CoverageReportGenerator --presubmit "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" \
+--token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" \
+--log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" \
+--pull-request-num ${{github.event.pull_request.number}} --base-commit "$base_commit"
+
+```
+
+### Add new coverage data to the storage of Metrics Service
+
+In a workflow, this will run in merge events or postsubmit tests. After each merge, all pod tests
+will run to add a new commit and its corresponding coverage data.
+```
+swift run CoverageReportGenerator --merge "firebase/firebase-ios-sdk" --commit "${GITHUB_SHA}" \
+--token $(gcloud auth print-identity-token) --xcresult-dir "/Users/runner/test/codecoverage" \
+--log-link "https://github.com/firebase/firebase-ios-sdk/actions/runs/${GITHUB_RUN_ID}" --branch \
+"${GITHUB_REF##*/}"
+```
+- `branch`: this is for merge and the new commit with coverage data will be linked with the branch
+in the database of Metrics Service.
+
+### Details
+
+More details in go/firebase-ios-sdk-test-coverage-metrics. Can also run
+`swift run CoverageReportGenerator -h` for help info.