Backport some CI network reliability changes (#12011)

This backports a blending of #10808 and #11532 to the 24.0.x release
branch to try and make CI a bit more reliable.

Author: alexcrichton

.github/actions/install-rust/action.yml

@@ -13,6 +13,10 @@ runs:
     - name: Install Rust
       shell: bash
       id: select
+      env:
+        # This is an attempt to reduce flakiness and see what happens. If this
+        # lands it worked. Feel free to modify if this becomes flaky again.
+        RUSTUP_USE_CURL: 1
       run: |
         # Determine MSRV as N in `1.N.0` by looking at the `rust-version`
         # located in the root `Cargo.toml`.
@@ -32,7 +36,12 @@ runs:
       shell: bash
       run: |
         rustup set profile minimal
-        rustup update "${{ steps.select.outputs.version }}" --no-self-update
+        for attempt in 1 2 3 4 5; do
+          [ "$attempt" = "5" ] && exit 1
+          rustup update "${{ steps.select.outputs.version }}" --no-self-update \
+            && break
+          sleep 5
+        done
         rustup default "${{ steps.select.outputs.version }}"
 
         # Save disk space by avoiding incremental compilation. Also turn down

.github/workflows/main.yml

@@ -109,7 +109,9 @@ jobs:
     - uses: ./.github/actions/install-rust
     - run: |
         set -e
-        curl -L https://github.com/EmbarkStudios/cargo-deny/releases/download/0.14.5/cargo-deny-0.14.5-x86_64-unknown-linux-musl.tar.gz | tar xzf -
+        curl --retry 5 --retry-all-errors -L -o cargo-deny.tar.gz https://github.com/EmbarkStudios/cargo-deny/releases/download/0.14.5/cargo-deny-0.14.5-x86_64-unknown-linux-musl.tar.gz
+        tar xzf cargo-deny.tar.gz
+        rm cargo-deny.tar.gz
         mv cargo-deny-*-x86_64-unknown-linux-musl/cargo-deny cargo-deny
         echo `pwd` >> $GITHUB_PATH
     - run: cargo deny check bans licenses
@@ -277,7 +279,10 @@ jobs:
         toolchain: wasmtime-ci-pinned-nightly
 
     # Build C API documentation
-    - run: curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.3/doxygen-1.9.3.linux.bin.tar.gz/download | tar xzf -
+    - run: |
+        curl --retry 5 --retry-all-errors -o doxygen.tar.gz -L https://github.com/doxygen/doxygen/releases/download/Release_1_9_3/doxygen-1.9.3.linux.bin.tar.gz
+        tar xzf doxygen.tar.gz
+        rm doxygen.tar.gz
     - run: echo "`pwd`/doxygen-1.9.3/bin" >> $GITHUB_PATH
     - run: cmake -S crates/c-api -B target/c-api
     - run: cmake --build target/c-api --target doc
@@ -645,7 +650,9 @@ jobs:
         # way faster at arm emulation than the current version github actions'
         # ubuntu image uses. Disable as much as we can to get it to build
         # quickly.
-        curl https://download.qemu.org/qemu-$QEMU_BUILD_VERSION.tar.xz | tar xJf -
+        curl --retry 5 --retry-all-errors -o qemu.tar.xz https://download.qemu.org/qemu-$QEMU_BUILD_VERSION.tar.xz
+        tar xJf qemu.tar.xz
+        rm qemu.tar.xz
         cd qemu-$QEMU_BUILD_VERSION
         ./configure --target-list=${{ matrix.qemu_target }} --prefix=${{ runner.tool_cache}}/qemu --disable-tools --disable-slirp --disable-fdt --disable-capstone --disable-docs
         ninja -C build install
@@ -811,7 +818,9 @@ jobs:
 
     - name: Install wasm-tools
       run: |
-        curl -L https://github.com/bytecodealliance/wasm-tools/releases/download/wasm-tools-1.0.27/wasm-tools-1.0.27-x86_64-linux.tar.gz | tar xfz -
+        curl --retry 5 --retry-all-errors -o wasm-tools.tar.gz -L https://github.com/bytecodealliance/wasm-tools/releases/download/wasm-tools-1.0.27/wasm-tools-1.0.27-x86_64-linux.tar.gz
+        tar xfz wasm-tools.tar.gz
+        rm wasm-tools.tar.gz
         echo `pwd`/wasm-tools-1.0.27-x86_64-linux >> $GITHUB_PATH
 
     - run: ./ci/build-wasi-preview1-component-adapter.sh
@@ -955,7 +964,10 @@ jobs:
     - uses: ./.github/actions/install-rust
     - run: |
         cd ${{ runner.tool_cache }}
-        curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar xzf -
+        curl --retry 5 --retry-all-errors -L -o sccache.tar.gz \
+          https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz
+        tar xzf sccache.tar.gz
+        rm sccache.tar.gz
         echo "`pwd`/sccache-0.2.13-x86_64-unknown-linux-musl" >> $GITHUB_PATH
         echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV
     - run: rustc scripts/publish.rs

ci/docker/aarch64-linux/Dockerfile

@@ -4,7 +4,10 @@ RUN apt-get update -y && apt-get install -y gcc gcc-aarch64-linux-gnu ca-certifi
 
 # The CMake in Ubuntu 16.04 was a bit too old for us to use so download one from
 # CMake's own releases and use that instead.
-RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.29.3/cmake-3.29.3-linux-x86_64.tar.gz | tar xzf -
+RUN curl --retry 5 -L -o cmake.tar.gz \
+    https://github.com/Kitware/CMake/releases/download/v3.29.3/cmake-3.29.3-linux-x86_64.tar.gz && \
+  tar xf cmake.tar.gz && \
+  rm cmake.tar.gz
 ENV PATH=$PATH:/cmake-3.29.3-linux-x86_64/bin
 
 ENV PATH=$PATH:/rust/bin

ci/docker/s390x-linux/Dockerfile

@@ -4,7 +4,10 @@ RUN apt-get update -y && apt-get install -y gcc gcc-s390x-linux-gnu ca-certifica
 
 # The CMake in Ubuntu 16.04 was a bit too old for us to use so download one from
 # CMake's own releases and use that instead.
-RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.29.3/cmake-3.29.3-linux-x86_64.tar.gz | tar xzf -
+RUN curl --retry 5 -L -o cmake.tar.gz \
+    https://github.com/Kitware/CMake/releases/download/v3.29.3/cmake-3.29.3-linux-x86_64.tar.gz && \
+  tar xf cmake.tar.gz && \
+  rm cmake.tar.gz
 ENV PATH=$PATH:/cmake-3.29.3-linux-x86_64/bin
 
 ENV PATH=$PATH:/rust/bin