From 27b1cbafb25f40a2e1f6e77bc63d0e8bf1db7e43 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Wed, 13 Nov 2024 13:55:56 -0800 Subject: [PATCH 1/8] update sccache to latest version --- .ci/docker/common/install_cache.sh | 56 ++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index 45edb19b107..cd215f0b2da 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -12,6 +12,34 @@ set -ex # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" +install_ubuntu() { + echo "Preparing to build sccache from source" + apt-get update + # libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04. + # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh`` + apt-get install -y cargo + echo "Checking out sccache repo" + if [ -n "$CUDA_VERSION" ]; then + # TODO: Remove this + git clone https://github.com/pytorch/sccache + else + git clone https://github.com/mozilla/sccache -b v0.8.2 + fi + cd sccache + echo "Building sccache" + cargo build --release + cp target/release/sccache /opt/cache/bin + echo "Cleaning up" + cd .. + rm -rf sccache + apt-get remove -y cargo rustc + apt-get autoclean && apt-get clean + + echo "Downloading old sccache binary from S3 repo for PCH builds" + curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache-0.2.14a + chmod 755 /opt/cache/bin/sccache-0.2.14a +} + install_binary() { echo "Downloading sccache binary from S3 repo" curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache @@ -26,11 +54,33 @@ export PATH="/opt/cache/bin:$PATH" # https://github.com/pytorch/sccache has started failing mysteriously # in which sccache server couldn't start with the following error: # sccache: error: Invalid argument (os error 22) -install_binary +install_ubuntu function write_sccache_stub() { BINARY=$1 - printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n exec sccache %s \"\$@\"\nelse\n exec %s \"\$@\"\nfi" "$(which "${BINARY}")" "$(which "${BINARY}")" > "/opt/cache/bin/${BINARY}" + if [ $1 == "gcc" ]; then + # Do not call sccache recursively when dumping preprocessor argument + # For some reason it's very important for the first cached nvcc invocation + cat >"/opt/cache/bin/$1" <"/opt/cache/bin/$1" < /dev/null 2>&1 || true + as_ci_user sccache --stop-server >/dev/null 2>&1 || true rm -f "${SCCACHE_ERROR_LOG}" || true # Clear sccache stats before using it From d623db8cea4c8f6ca2accc550c8f5879c535d370 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Wed, 13 Nov 2024 15:11:45 -0800 Subject: [PATCH 2/8] + remove comment / binary download --- .ci/docker/common/install_cache.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index cd215f0b2da..8480278f68b 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -34,10 +34,6 @@ install_ubuntu() { rm -rf sccache apt-get remove -y cargo rustc apt-get autoclean && apt-get clean - - echo "Downloading old sccache binary from S3 repo for PCH builds" - curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache-0.2.14a - chmod 755 /opt/cache/bin/sccache-0.2.14a } install_binary() { @@ -50,10 +46,6 @@ mkdir -p /opt/cache/bin sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment export PATH="/opt/cache/bin:$PATH" -# NB: Install the pre-built binary from S3 as building from source -# https://github.com/pytorch/sccache has started failing mysteriously -# in which sccache server couldn't start with the following error: -# sccache: error: Invalid argument (os error 22) install_ubuntu function write_sccache_stub() { From 254d451f399fedf51781d6e5194bda19f7fbec85 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Thu, 14 Nov 2024 13:33:26 -0800 Subject: [PATCH 3/8] add default AWS region for SCCACHE --- .github/workflows/docker-builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index d256af7fcda..9a220a6f29b 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -26,6 +26,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true +env: + AWS_DEFAULT_REGION: us-east-1 + jobs: docker-build: runs-on: [self-hosted, linux.2xlarge] From 0e0157a8e111b5cd5a2383d8849ae0d7736b5d31 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Thu, 14 Nov 2024 13:49:21 -0800 Subject: [PATCH 4/8] add default AWS region for SCCACHE --- .ci/docker/common/install_cache.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index 8480278f68b..9f19375618f 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -79,6 +79,7 @@ EOF init_sccache() { # This is the remote cache bucket export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2 + export SCCACHE_REGION=us-east-1 export SCCACHE_S3_KEY_PREFIX=executorch export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_ERROR_LOG=/tmp/sccache_error.log From 8d60a320d7dcc2c36760043b2cbd94f6d3e45144 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Thu, 14 Nov 2024 14:49:16 -0800 Subject: [PATCH 5/8] add default AWS region for SCCACHE --- .ci/docker/common/install_cache.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index 9f19375618f..a3bbca14d8c 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -80,6 +80,7 @@ init_sccache() { # This is the remote cache bucket export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2 export SCCACHE_REGION=us-east-1 + export AWS_DEFAULT_REGION=us-east-1 export SCCACHE_S3_KEY_PREFIX=executorch export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_ERROR_LOG=/tmp/sccache_error.log From 15e08969495080a867738f5cdf9dd7acdc713d1b Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Thu, 14 Nov 2024 16:15:53 -0800 Subject: [PATCH 6/8] add region to dockerfile --- .ci/docker/ubuntu/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index f55504c9700..713a076d617 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -57,6 +57,7 @@ COPY ./common/utils.sh utils.sh RUN bash ./install_cache.sh && rm install_cache.sh utils.sh ENV SCCACHE_BUCKET ossci-compiler-cache-circleci-v2 ENV SCCACHE_S3_KEY_PREFIX executorch +ENV SCCACHE_REGION us-east-1 ARG TORCH_VERSION COPY ./common/install_pytorch.sh install_pytorch.sh From 46fdf75678784fd944c1baf0cf1ddfb1a39160cc Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Fri, 15 Nov 2024 09:07:51 -0800 Subject: [PATCH 7/8] probablyl don't need region in init --- .ci/docker/common/install_cache.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index a3bbca14d8c..8480278f68b 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -79,8 +79,6 @@ EOF init_sccache() { # This is the remote cache bucket export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2 - export SCCACHE_REGION=us-east-1 - export AWS_DEFAULT_REGION=us-east-1 export SCCACHE_S3_KEY_PREFIX=executorch export SCCACHE_IDLE_TIMEOUT=0 export SCCACHE_ERROR_LOG=/tmp/sccache_error.log From bbee49c6cd0530638f26f288a8d3f765cf251b6f Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Fri, 15 Nov 2024 11:57:25 -0800 Subject: [PATCH 8/8] remove nvidia check --- .ci/docker/common/install_cache.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.ci/docker/common/install_cache.sh b/.ci/docker/common/install_cache.sh index 8480278f68b..f17b4cc6068 100755 --- a/.ci/docker/common/install_cache.sh +++ b/.ci/docker/common/install_cache.sh @@ -19,12 +19,8 @@ install_ubuntu() { # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh`` apt-get install -y cargo echo "Checking out sccache repo" - if [ -n "$CUDA_VERSION" ]; then - # TODO: Remove this - git clone https://github.com/pytorch/sccache - else - git clone https://github.com/mozilla/sccache -b v0.8.2 - fi + git clone https://github.com/mozilla/sccache -b v0.8.2 + cd sccache echo "Building sccache" cargo build --release