ci: parallelize API report generation (#10269)

Run `abi-dumper; abi-compliance-checker` in parallel for each library.
This reduces the run time of "check-api" (in a totally unscientific,
one-off, "ninja: no work to do" test) from around 20m30s to about 7m30s.

Also take this opportunity to improve the output by discarding the
noise ("cmake --install" messages, "missed type id" errors, etc.) and
by including the elapsed time for each library.

On the latter front, the outliers are ...
```
bigtable completed in 4m2s
dialogflow_cx completed in 6m7s
dialogflow_es completed in 6m56s
```
where we can see that the "6m56" dominates the overall run time.

Fixes #10234

Author: devbww

ci/cloudbuild/builds/check-api.sh

@@ -25,9 +25,13 @@ mapfile -t cmake_args < <(cmake::common_args)
 
 mapfile -t feature_list < <(bazelisk --batch query \
   --noshow_progress --noshow_loading_progress \
-  'kind(cc_library, //:all) except filter("experimental|mocks", kind(cc_library, //:all))' |
+  'kind(cc_library, //:all)
+   except filter("experimental|mocks", kind(cc_library, //:all))' |
   sed -e 's;//:;;')
 enabled="$(printf ";%s" "${feature_list[@]}")"
+# These two are not libraries that require enabling.
+enabled="${enabled/;common/}"
+enabled="${enabled/;grpc_utils/}"
 enabled="${enabled:1}"
 
 INSTALL_PREFIX=/var/tmp/google-cloud-cpp
@@ -43,30 +47,37 @@ cmake "${cmake_args[@]}" \
   -DGOOGLE_CLOUD_CPP_ENABLE="${enabled}" \
   -DCMAKE_CXX_FLAGS="-Og -Wno-maybe-uninitialized"
 cmake --build cmake-out
-cmake --install cmake-out
+cmake --install cmake-out >/dev/null
 
-# Uses `abi-dumper` to dump the ABI for the given library, which should be
-# installed at the given prefix. This function will be called from a subshell,
-# so it cannot use other variables or functions (including io::log*).
-function dump_abi() {
+# Uses `abi-dumper` to dump the ABI for the given library, which should
+# be installed at the given @p prefix, and `abi-compliance-checker` to
+# produce a report comparing the old and new dumps. The (compressed) new
+# dump is left in the @p project_root tree.
+#
+# This function will be called from a subshell, so it cannot use other
+# variables or functions (including io::log*).
+function check_abi() {
   local library="$1"
   local prefix="$2"
-  local public_headers="${prefix}/include/google/cloud/${library#google_cloud_cpp_}"
-  if [[ "${library}" == "google_cloud_cpp_common" || "${library}" == "google_cloud_cpp_grpc_utils" ]]; then
-    # These two are special
+  local project_root="$3"
+
+  local shortlib="${library#google_cloud_cpp_}"
+  local public_headers="${prefix}/include/google/cloud/${shortlib}"
+  # These two are special
+  if [[ "${shortlib}" == "common" || "${shortlib}" == "grpc_utils" ]]; then
     public_headers="${prefix}/include/google/cloud"
   fi
 
-  echo "Dumping ${library} (may be slow)..."
   local version
   version=$(git rev-parse --short HEAD)
+  local actual_dump_file="${library}.actual.abi.dump"
   local -a dump_options=(
     # The source .so file
     "${prefix}/lib64/lib${library}.so"
     # Use the git version as the library version number for reporting purposes
     -lver "${version}"
     # The dump destination
-    -o "cmake-out/${library}.actual.abi.dump"
+    -o "cmake-out/${actual_dump_file}"
     # Where to find the headers
     -include-paths "${prefix}/include"
     -include-paths "/usr/local/include"
@@ -81,57 +92,65 @@ function dump_abi() {
     # Use the system's debuginfo
     -search-debuginfo /usr
   )
-  abi-dumper "${dump_options[@]}"
-}
-export -f dump_abi # enables this function to be called from a subshell
-
-mapfile -t libraries < <(printf "google_cloud_cpp_%s\n" "${feature_list[@]}")
-
-# Run the dump_abi function for each library in parallel since its slow.
-echo "${libraries[@]}" | xargs -P "$(nproc)" -n 1 \
-  bash -c "dump_abi \$0 ${INSTALL_PREFIX}"
-
-# A count of the number of libraries that fail the api compliance check.
-# This will become the script's exit code.
-errors=0
+  abi-dumper "${dump_options[@]}" >/dev/null 2>&1 |
+    grep -v "ERROR: missed type id" || true
 
-for lib in "${libraries[@]}"; do
-  io::log_h2 "Checking ${lib}"
-  actual_dump_file="${lib}.actual.abi.dump"
-  expected_dump_file="${lib}.expected.abi.dump"
-  expected_dump_path="${PROJECT_ROOT}/ci/abi-dumps/${expected_dump_file}.gz"
+  local project_dir="${project_root}/ci/abi-dumps"
+  local expected_dump_file="${library}.expected.abi.dump"
+  local expected_dump_path="${project_dir}/${expected_dump_file}.gz"
   if [[ -r "${expected_dump_path}" ]]; then
     zcat "${expected_dump_path}" >"cmake-out/${expected_dump_file}"
-    report="cmake-out/compat_reports/${lib}/src_compat_report.html"
+    report="cmake-out/compat_reports/${library}/src_compat_report.html"
     compliance_flags=(
       # Put the output report in a separate directory for each library
       -report-path "${report}"
       # We only want a source-level report. We make no ABI guarantees, such
       # as data structure sizes or virtual table ordering
       -src
-      # We ignore all symbols in internal namespaces, because these are not part
-      # of our public API. We do this by specifying a regex that matches against
-      # the mangled symbol names. For example, 8 is the number of characters in
-      # the string "internal", and it should again be followed by some other
-      # number indicating the length of the symbol within the "internal"
-      # namespace. See: https://en.wikipedia.org/wiki/Name_mangling
+      # We ignore all symbols in internal namespaces, because these are not
+      # part of our public API. We do this by specifying a regex that matches
+      # against the mangled symbol names. For example, 8 is the number of
+      # characters in the string "internal", and it should again be followed
+      # by some other number indicating the length of the symbol within the
+      # "internal" namespace. See: https://en.wikipedia.org/wiki/Name_mangling
       -skip-internal-symbols "(8internal|_internal)\d"
       # The library to compare
-      -l "${lib}"
+      -l "${library}"
       # Compared the saved baseline vs. the dump for the current version
       -old "cmake-out/${expected_dump_file}"
       -new "cmake-out/${actual_dump_file}"
     )
-    if ! io::run abi-compliance-checker "${compliance_flags[@]}"; then
-      io::log_red "ABI Compliance error: ${lib}"
-      ((++errors))
-      io::log "Report file: ${report}"
-      w3m -dump "${report}"
-    fi
+    abi-compliance-checker "${compliance_flags[@]}" >/dev/null || true
   fi
+
   # Replaces the (old) expected dump file with the (new) actual one.
   gzip -n "cmake-out/${actual_dump_file}"
   mv -f "cmake-out/${actual_dump_file}.gz" "${expected_dump_path}"
+}
+export -f check_abi # enables this function to be called from a subshell
+
+mapfile -t libraries < <(printf "google_cloud_cpp_%s\n" "${feature_list[@]}")
+
+# Run the check_abi function for each library in parallel since its slow.
+echo "${libraries[@]}" | xargs -P "$(nproc)" -n 1 \
+  bash -c "TIMEFORMAT=\"\${0#google_cloud_cpp_} completed in %0lR\";
+           time check_abi \${0} ${INSTALL_PREFIX} ${PROJECT_ROOT}"
+
+# A count of the number of libraries that fail the api compliance check.
+# This will become the script's exit code.
+errors=0
+
+for library in "${libraries[@]}"; do
+  report="cmake-out/compat_reports/${library}/src_compat_report.html"
+  if grep --silent "<td class='compatible'>100%</td>" "${report}"; then
+    io::log_green "ABI Compliance OK: ${library}"
+  else
+    io::log_red "ABI Compliance error: ${library}"
+    io::log "Report file: ${report}"
+    w3m -dump "${report}"
+    ((++errors))
+  fi
 done
+
 echo
 exit "${errors}"

ci/generate-markdown/generate-readme.sh

@@ -36,7 +36,7 @@ file="README.md"
 (
   mapfile -t libraries < <(bazelisk --batch query \
     --noshow_progress --noshow_loading_progress \
-    'kind(cc_library, //:all) except filter("experimental|mocks|common|grpc_utils|internal", kind(cc_library, //:all))' |
+    'kind(cc_library, //:all) except filter("experimental|mocks|common|grpc_utils", kind(cc_library, //:all))' |
     sed -e 's;//:;;' |
     LC_ALL=C sort)
   sed '/<!-- inject-GA-libraries-start -->/q' "${file}"