Rollup merge of rust-lang#63485 - pietroalbini:new-mirror-bucket, r=alexcrichton

Mark-Simulacrum · web-flow · commit e50b9438802a · 2019-08-13T20:26:47.000-04:00
ci: move mirrors to their standalone bucket Currently mirrors are stored in the rust-lang-ci2 S3 bucket along with CI toolchains. This is problematic for multiple reasons: - CI IAM credentials are allowed to both edit and delete those files. A malicious user gaining access to those credentials would be able to change our mirrored dependencies, possibly compromising the compiler. - Contents of the rust-lang-ci2 bucket are disposable except for the mirrors' content. When we implement backups for S3 buckets we'd have to replicate just that part of the bucket, complicating the backup logic and increasing the chance of mistakes. A standalone bucket will be way easier to backup. This commit switches our CI to use the new rust-lang-ci-mirrors bucket. r? @alexcrichton
diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
@@ -273,7 +273,7 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
         SCRIPT: make ci-subset-1
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
         # FIXME(#59637)
@@ -283,14 +283,14 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
         SCRIPT: make ci-subset-2
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
       x86_64-mingw-1:
         MSYS_BITS: 64
         SCRIPT: make ci-subset-1
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
         # FIXME(#59637)
@@ -300,7 +300,7 @@ jobs:
         MSYS_BITS: 64
         SCRIPT: make ci-subset-2
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
 
@@ -327,7 +327,7 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
         SCRIPT: python x.py dist
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
         DIST_REQUIRE_ALL_TOOLS: 1
@@ -336,7 +336,7 @@ jobs:
         MSYS_BITS: 64
         SCRIPT: python x.py dist
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
         DIST_REQUIRE_ALL_TOOLS: 1
diff --git a/src/ci/azure-pipelines/steps/install-clang.yml b/src/ci/azure-pipelines/steps/install-clang.yml
@@ -36,7 +36,7 @@ steps:
     set -e
     mkdir -p citools
     cd citools
-    curl -f https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/LLVM-7.0.0-win64.tar.gz | tar xzf -
+    curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/LLVM-7.0.0-win64.tar.gz | tar xzf -
     echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --set llvm.clang-cl=`pwd`/clang-rust/bin/clang-cl.exe"
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],''))
   displayName: Install clang (Windows)
diff --git a/src/ci/azure-pipelines/steps/install-sccache.yml b/src/ci/azure-pipelines/steps/install-sccache.yml
@@ -2,14 +2,14 @@ steps:
 
 - bash: |
     set -e
-    curl -fo /usr/local/bin/sccache https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
+    curl -fo /usr/local/bin/sccache https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-apple-darwin
     chmod +x /usr/local/bin/sccache
   displayName: Install sccache (OSX)
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
 
 - script: |
     md sccache
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-26-sccache-x86_64-pc-windows-msvc"
     echo ##vso[task.prependpath]%CD%\sccache
   displayName: Install sccache (Windows)
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml
@@ -4,7 +4,7 @@ steps:
 # https://github.com/wixtoolset/wix3 originally
 - bash: |
     set -e
-    curl -O https://rust-lang-ci2.s3-us-west-1.amazonaws.com/rust-ci-mirror/wix311-binaries.zip
+    curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip
     echo "##vso[task.setvariable variable=WIX]`pwd`/wix"
     mkdir -p wix/bin
     cd wix/bin
@@ -18,7 +18,7 @@ steps:
 # one is MSI installers and one is EXE, but they're not used so frequently at
 # this point anyway so perhaps it's a wash!
 - script: |
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-08-22-is.exe"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe"
     is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-
     echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5
   displayName: Install InnoSetup
@@ -109,7 +109,7 @@ steps:
 # Note that this is originally from the github releases patch of Ninja
 - script: |
     md ninja
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-03-15-ninja-win.zip"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip"
     7z x -oninja 2017-03-15-ninja-win.zip
     del 2017-03-15-ninja-win.zip
     set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja
diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile
@@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static
 
 # TODO: What is this?!
 # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb
-RUN curl -O https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/vexpress-v2p-ca15-tc1.dtb
+RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh
@@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl
 # originally from
 # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/
 # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2
-URL="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror"
+URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc"
 FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2"
 curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2
 
diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Originally from https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-curl https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
+curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
   tar xJf -
 export PATH=`pwd`/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH
 
diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh
@@ -4,7 +4,7 @@ set -ex
 source shared.sh
 
 VERSION=1.0.2k
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/openssl-$VERSION.tar.gz
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz
 
 curl $URL | tar xzf -
 
diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
@@ -25,7 +25,7 @@ cd netbsd
 
 mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot
 
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
 
 # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz
 curl $URL/2018-03-01-netbsd-src.tgz | tar xzf -
diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py
@@ -23,8 +23,9 @@
 HOST_OS = "linux"
 
 # Mirroring options
-MIRROR_BUCKET = "rust-lang-ci2"
-MIRROR_BASE_DIR = "rust-ci-mirror/android/"
+MIRROR_BUCKET = "rust-lang-ci-mirrors"
+MIRROR_BUCKET_REGION = "us-west-1"
+MIRROR_BASE_DIR = "rustc/android/"
 
 import argparse
 import hashlib
@@ -144,7 +145,8 @@ def cli_install(args):
     lockfile = Lockfile(args.lockfile)
     for package in lockfile.packages.values():
         # Download the file from the mirror into a temp file
-        url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
+        url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \
+              ".amazonaws.com/" + MIRROR_BASE_DIR
         downloaded = package.download(url)
         # Extract the file in a temporary directory
         extract_dir = tempfile.mkdtemp()
diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh
@@ -59,7 +59,7 @@ done
 
 # Originally downloaded from:
 # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
 curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}"
 
 # Fix up absolute symlinks from the system image.  This can be removed
diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh
@@ -1,6 +1,6 @@
 set -ex
 
 curl -fo /usr/local/bin/sccache \
-  https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
+  https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl
 
 chmod +x /usr/local/bin/sccache
diff --git a/src/ci/install-awscli.sh b/src/ci/install-awscli.sh
@@ -16,7 +16,7 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-MIRROR="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-07-27-awscli.tar"
+MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar"
 DEPS_DIR="/tmp/awscli-deps"
 
 pip="pip"
