ci: deflake macOS builds with CMake (#9306)

I searched for places where there is an explicit or implicit download,
and wrap them with a retry loop. This motivated me to refactor the
code to install vcpkg.

Incidentally, my previous changes to use GCS for the vcpkg cache may
have increased flakiness: instead of one download for the vcpkg binary
caches it performs N (currently 14). This is fixed in this change too.

Author: coryan

ci/kokoro/lib/vcpkg.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make our include guard clean against set -o nounset.
+test -n "${CI_KOKORO_LIB_VCPKG_SH__:-}" || declare -i CI_KOKORO_LIB_VCPKG_SH__=0
+if ((CI_KOKORO_LIB_VCPKG_SH__++ != 0)); then
+  return 0
+fi # include guard
+
+source module /ci/lib/io.sh
+source module /ci/kokoro/lib/cache.sh
+source module /ci/kokoro/lib/gcloud.sh
+
+install_vcpkg() {
+  local -r vcpkg_dir="$1"
+
+  mkdir -p "${vcpkg_dir}"
+  io::log "Downloading vcpkg into ${vcpkg_dir}..."
+  VCPKG_COMMIT="$(<ci/etc/vcpkg-commit.txt)"
+  ci/retry-command.sh 3 120 curl -sSL "https://github.com/microsoft/vcpkg/archive/${VCPKG_COMMIT}.tar.gz" |
+    tar -C "${vcpkg_dir}" --strip-components=1 -zxf -
+
+  io::log_h2 "Configure VCPKG to use GCS as a cache"
+  readonly CACHE_BUCKET="${GOOGLE_CLOUD_CPP_KOKORO_RESULTS:-cloud-cpp-kokoro-results}"
+  readonly CACHE_FOLDER="${CACHE_BUCKET}/build-cache/google-cloud-cpp/vcpkg/macos"
+  export VCPKG_BINARY_SOURCES="x-gcs,gs://${CACHE_BUCKET}/${CACHE_FOLDER},readwrite"
+
+  create_gcloud_config
+  activate_service_account_keyfile "${CACHE_KEYFILE}"
+  export CLOUDSDK_ACTIVE_CONFIG_NAME="${GCLOUD_CONFIG}"
+  io::run gsutil ls "gs://${CACHE_BUCKET}/"
+  # Eventually we can remove this, but the current caches contain both `ccache`
+  # files (which we want), and `vcpkg` files (which we don't want).
+  io::run rm -fr "${HOME}/.cache/vcpkg"
+
+  ci/retry-command.sh 3 120 "${vcpkg_dir}/bootstrap-vcpkg.sh"
+  "${vcpkg_dir}/vcpkg" --feature-flags=-manifests remove --outdated --recurse
+}

ci/kokoro/macos/builds/cmake-vcpkg.sh

@@ -19,35 +19,26 @@ set -euo pipefail
 source "$(dirname "$0")/../../../lib/init.sh"
 source module ci/etc/integration-tests-config.sh
 source module ci/lib/io.sh
+source module ci/kokoro/lib/vcpkg.sh
 
 readonly SOURCE_DIR="."
 readonly BINARY_DIR="cmake-out/macos-vcpkg"
 
 NCPU="$(sysctl -n hw.logicalcpu)"
 readonly NCPU
 
-io::log_h2 "Using CMake version"
-cmake --version
-
 io::log_h2 "Update or install dependencies"
 brew list --versions openssl || ci/retry-command.sh 3 120 brew install openssl
 brew list --versions ccache || ci/retry-command.sh 3 120 brew install ccache
 brew list --versions cmake || ci/retry-command.sh 3 120 brew install cmake
 brew list --versions ninja || ci/retry-command.sh 3 120 brew install ninja
 
+io::log_h2 "Using CMake version"
+cmake --version
+
 # Fetch vcpkg at the specified hash.
 vcpkg_dir="${HOME}/vcpkg-quickstart"
-mkdir -p "${vcpkg_dir}"
-io::log "Downloading vcpkg into ${vcpkg_dir}..."
-VCPKG_COMMIT="$(<ci/etc/vcpkg-commit.txt)"
-ci/retry-command.sh 3 120 curl -sSL "https://github.com/microsoft/vcpkg/archive/${VCPKG_COMMIT}.tar.gz" |
-  tar -C "${vcpkg_dir}" --strip-components=1 -zxf -
-(
-  cd "${vcpkg_dir}"
-  env VCPKG_ROOT="${vcpkg_dir}" CC="ccache cc" CXX="ccache c++" \
-    "${PROJECT_ROOT}/ci/retry-command.sh" 3 120 ./bootstrap-vcpkg.sh
-  ./vcpkg remove --outdated --recurse
-)
+install_vcpkg "${vcpkg_dir}"
 
 io::log_h2 "ccache stats"
 ccache --show-stats
@@ -60,23 +51,28 @@ export OPENSSL_ROOT_DIR="${homebrew_prefix}/opt/openssl"
 
 cmake_flags=(
   "-DCMAKE_TOOLCHAIN_FILE=${vcpkg_dir}/scripts/buildsystems/vcpkg.cmake"
-  "-DCMAKE_INSTALL_PREFIX=$HOME/staging"
+  "-DCMAKE_INSTALL_PREFIX=${HOME}/staging"
   "-DGOOGLE_CLOUD_CPP_ENABLE_CCACHE=ON"
 )
 
+# The downloads can fail, therefore require a retry loop.
+io::log_h2 "Download and compile dependencies using vcpkg"
+ci/retry-command.sh 3 120 \
+  "${vcpkg_dir}/vcpkg" install --x-install-root="${BINARY_DIR}/vcpkg_installed"
+
 io::log_h2 "Configure CMake"
 export NINJA_STATUS="T+%es [%f/%t] "
-cmake -GNinja "-H${SOURCE_DIR}" "-B${BINARY_DIR}" "${cmake_flags[@]}"
+cmake -GNinja -S "${SOURCE_DIR}" -B "${BINARY_DIR}" "${cmake_flags[@]}"
 
-io::log_h2 "Compiling with ${NCPU} cpus"
-cmake --build "${BINARY_DIR}" -- -j "${NCPU}"
+io::log_h2 "Compiling using all CPUs"
+cmake --build "${BINARY_DIR}"
 
 io::log_h2 "Running unit tests"
 ctest_args=(
   # Print the full output on failures
   --output-on-failure
   # Run many tests in parallel, use -j for compatibility with old versions
-  -j "$(nproc)"
+  -j "${NCPU}"
   # Make the output shorter on interactive tests
   --progress
 )

ci/kokoro/macos/builds/quickstart-cmake.sh

@@ -20,8 +20,7 @@ source "$(dirname "$0")/../../../lib/init.sh"
 source module ci/lib/io.sh
 source module ci/etc/integration-tests-config.sh
 source module ci/etc/quickstart-config.sh
-source module ci/kokoro/lib/gcloud.sh
-source module ci/kokoro/lib/cache.sh
+source module ci/kokoro/lib/vcpkg.sh
 
 io::log_h2 "Using CMake version"
 cmake --version
@@ -32,32 +31,13 @@ brew list --versions pkg-config || brew install pkg-config
 
 # Fetch vcpkg at the specified hash.
 vcpkg_dir="${HOME}/vcpkg-quickstart"
-mkdir -p "${vcpkg_dir}"
-io::log "Downloading vcpkg into ${vcpkg_dir}..."
-VCPKG_COMMIT="$(<ci/etc/vcpkg-commit.txt)"
-
-ci/retry-command.sh 3 120 curl -sSL "https://github.com/microsoft/vcpkg/archive/${VCPKG_COMMIT}.tar.gz" |
-  tar -C "${vcpkg_dir}" --strip-components=1 -zxf -
-
-io::log_h2 "Configure VCPKG to use GCS as a cache"
-readonly CACHE_BUCKET="${GOOGLE_CLOUD_CPP_KOKORO_RESULTS:-cloud-cpp-kokoro-results}"
-readonly CACHE_FOLDER="${CACHE_BUCKET}/build-cache/google-cloud-cpp/vcpkg/macos"
-export VCPKG_BINARY_SOURCES="x-gcs,gs://${CACHE_BUCKET}/${CACHE_FOLDER},readwrite"
-
-create_gcloud_config
-activate_service_account_keyfile "${CACHE_KEYFILE}"
-export CLOUDSDK_ACTIVE_CONFIG_NAME="${GCLOUD_CONFIG}"
-io::run gsutil ls "gs://${CACHE_BUCKET}/"
-# Eventually we can remove this, but the current caches contain both `ccache`
-# files (which we want), and `vcpkg` files (which we don't want).
-io::run rm -fr "${HOME}/.cache/vcpkg"
+install_vcpkg "${vcpkg_dir}"
 
+# The downloads can fail, therefore require a retry loop.
 (
   cd "${vcpkg_dir}"
-  env VCPKG_ROOT="${vcpkg_dir}" CC="ccache cc" CXX="ccache c++" \
-    "${PROJECT_ROOT}/ci/retry-command.sh" 3 120 ./bootstrap-vcpkg.sh
-  ./vcpkg remove --outdated --recurse
-  ./vcpkg install google-cloud-cpp
+  "${PROJECT_ROOT}/ci/retry-command.sh" 3 120 \
+    "${vcpkg_dir}/vcpkg" "--feature-flags=-manifests" install google-cloud-cpp
 )
 
 export NINJA_STATUS="T+%es [%f/%t] "
@@ -76,17 +56,17 @@ build_quickstart() {
 
   io::log_h2 "Configure CMake for ${library}'s quickstart"
   "${cmake}" "-GNinja" "-DCMAKE_MAKE_PROGRAM=${ninja}" \
-    "-H${source_dir}" "-B${binary_dir}" "${cmake_flags[@]}"
+    -S "${source_dir}" -B "${binary_dir}" "${cmake_flags[@]}"
 
   echo
   io::log_h2 "Compiling ${library}'s quickstart"
   "${cmake}" --build "${binary_dir}"
 }
 
 io::log_h2 "Fetch vcpkg tools"
-env VCPKG_ROOT="${vcpkg_dir}" "${PROJECT_ROOT}/ci/retry-command.sh" 3 120 \
+ci/retry-command.sh 3 120 \
   "${vcpkg_dir}/vcpkg" "--feature-flags=-manifests" fetch cmake
-env VCPKG_ROOT="${vcpkg_dir}" "${PROJECT_ROOT}/ci/retry-command.sh" 3 120 \
+ci/retry-command.sh 3 120 \
   "${vcpkg_dir}/vcpkg" "--feature-flags=-manifests" fetch ninja
 
 errors=""