Merge bitcoin/bitcoin#23597: ci: Revamp SDK caches

6c7b883 ci: Always run merge_base_script first for consistency (Hennadii Stepanov)
3ad5ace ci: Revamp Android SDK cache, and update it timely (Hennadii Stepanov)
e988bc7 ci: Make macOS SDK cache independent, and update it timely (Hennadii Stepanov)

Pull request description:

  On master (681b25e) SDK (macOS and Android) are not invalidated timely.
  For instance, during updating macOS SDK in #21851 it is not possible to run a related CI task successfully.

  Another flaw is related to Android SDK -- the following commands https://github.com/bitcoin/bitcoin/blob/681b25e3cd7d084f642693152322ed9a40f33ba0/ci/test/05_before_script.sh#L25-L32 run regardless of cache state. See CI task [log](https://api.cirrus-ci.com/v1/task/5962074015662080/logs/ci.log) for details.

  This PR:
  - drops caching of `depends/sdk-sources` in favor of separated caching of `depends/SDKs/$MACOS_SDK` and `depends/SDKs/android`
  - establishes conditions for cache invalidation
  - bypass unneeded commands when a cache is populated

ACKs for top commit:
  fanquake:
    ACK 6c7b883

Tree-SHA512: e5a5ebefd47fad495f08b8f821573977c1b8a906b3424396973bdf41da255446edbcbd43ea4f4f837f77ac95380f32e436eb75944f3457ec995057445b3b215e

Author: fanquake

.cirrus.yml

@@ -51,10 +51,6 @@ global_task_template: &GLOBAL_TASK_TEMPLATE
   << : *BASE_TEMPLATE
   << : *MAIN_TEMPLATE
 
-depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE
-  depends_sdk_cache:
-    folder: "depends/sdk-sources"
-
 compute_credits_template: &CREDITS_TEMPLATE
   # https://cirrus-ci.org/pricing/#compute-credits
   # Only use credits for pull requests to the main repo
@@ -273,11 +269,15 @@ task:
 
 task:
   name: 'macOS 10.15 [gui, no tests] [focal]'
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
-  << : *GLOBAL_TASK_TEMPLATE
+  << : *BASE_TEMPLATE
+  macos_sdk_cache:
+    folder: "depends/SDKs/$MACOS_SDK"
+    fingerprint_key: "$MACOS_SDK"
+  << : *MAIN_TEMPLATE
   container:
     image: ubuntu:focal
   env:
+    MACOS_SDK: "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers"
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_mac.sh"
 
@@ -297,8 +297,10 @@ task:
 
 task:
   name: 'ARM64 Android APK [focal]'
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
   << : *BASE_TEMPLATE
+  android_sdk_cache:
+    folder: "depends/SDKs/android"
+    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=22.1.7171670"
   depends_sources_cache:
     folder: "depends/sources"
     fingerprint_script: git rev-list -1 HEAD ./depends

ci/test/05_before_script.sh

@@ -15,17 +15,22 @@ fi
 
 DOCKER_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
 
-OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
 
-if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
-  DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
+if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
+  OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
+  OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
+  if [ ! -f "$OSX_SDK_PATH" ]; then
+    DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
+  fi
+  DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
 fi
 
-if [ -n "$ANDROID_TOOLS_URL" ]; then
-  ANDROID_TOOLS_PATH=$DEPENDS_DIR/sdk-sources/android-tools.zip
-
-  DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
+if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
+  ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
+  if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
+    DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
+  fi
   DOCKER_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools"
   DOCKER_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools"
   DOCKER_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
@@ -38,9 +43,6 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 fi
 
-if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
-  DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
-fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set "${HOST}-g++" \$\(which "${HOST}-g++-posix"\)
 fi