Upload on PR builds as well (#363)

staticfloat · web-flow · commit fc0448752df1 · 2024-06-19T20:52:56.000-07:00
* Upload on PR builds as well This is a follow-up to c8b4709, I forgot that we elide the upload jobs completely when not on `master` or `release-*`. This remediates the situation, uploading PR builds as well, but not codesigning them. * sign treehashes
diff --git a/pipelines/main/launch_signed_jobs.yml.signature b/pipelines/main/launch_signed_jobs.yml.signature
@@ -1 +1,2 @@
-Salted__��*��8��ũ.O���ʶ��S����I�"�����X.Q٘oKS��(�		��T��w&��k?_s����_I�%��ߙ;~0�
+Salted__V:l*SY'��rc�\b���9�T�nX ��vUd"�w�~0B����0�������&����7�^
+|IKg�v<;T�M��H���	
diff --git a/pipelines/main/launch_upload_jobs.yml.signature b/pipelines/main/launch_upload_jobs.yml.signature
diff --git a/pipelines/main/platforms/upload_freebsd.yml b/pipelines/main/platforms/upload_freebsd.yml
@@ -3,12 +3,6 @@ steps:
     steps:
       - label: ":freebsd: upload ${TRIPLET?}"
         key:   "upload_${TRIPLET?}"
-        # We only upload to S3 if one of the following criteria are true:
-        # 1. The branch is `master`.
-        # 2. The branch is `release-*`.
-        # 3. The build is a tag build AND the tag is `v*`.
-        # 4. The pipeline is `julia-buildkite`.
-        if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
         depends_on:
           # Wait for the builder to finish
           - "build_${TRIPLET?}"
diff --git a/pipelines/main/platforms/upload_linux.yml b/pipelines/main/platforms/upload_linux.yml
@@ -3,12 +3,6 @@ steps:
     steps:
       - label: ":linux: upload ${TRIPLET?}"
         key:   "upload_${TRIPLET?}"
-        # We only upload to S3 if one of the following criteria are true:
-        # 1. The branch is `master`.
-        # 2. The branch is `release-*`.
-        # 3. The build is a tag build AND the tag is `v*`.
-        # 4. The pipeline is `julia-buildkite`.
-        if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
         depends_on:
           # Wait for the builder to finish
           - "build_${TRIPLET?}"
diff --git a/pipelines/main/platforms/upload_macos.yml b/pipelines/main/platforms/upload_macos.yml
@@ -3,12 +3,6 @@ steps:
     steps:
       - label: ":macos: upload ${TRIPLET?}"
         key:   "upload_${TRIPLET?}"
-        # We only upload to S3 if one of the following criteria are true:
-        # 1. The branch is `master`.
-        # 2. The branch is `release-*`.
-        # 3. The build is a tag build AND the tag is `v*`.
-        # 4. The pipeline is `julia-buildkite`.
-        if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
         depends_on:
           # Wait for the builder to finish
           - "build_${TRIPLET?}"
diff --git a/pipelines/main/platforms/upload_windows.yml b/pipelines/main/platforms/upload_windows.yml
@@ -3,12 +3,6 @@ steps:
     steps:
       - label: ":windows: upload ${TRIPLET?}"
         key:   "upload_${TRIPLET?}"
-        # We only upload to S3 if one of the following criteria are true:
-        # 1. The branch is `master`.
-        # 2. The branch is `release-*`.
-        # 3. The build is a tag build AND the tag is `v*`.
-        # 4. The pipeline is `julia-buildkite`.
-        if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
         depends_on:
           # Wait for the builder to finish
           - "build_${TRIPLET?}"
diff --git a/pipelines/scheduled/coverage/coverage.yml.signature b/pipelines/scheduled/coverage/coverage.yml.signature
diff --git a/pipelines/scheduled/launch_signed_jobs.yml.signature b/pipelines/scheduled/launch_signed_jobs.yml.signature
@@ -1,3 +1 @@
-Salted__�R��e��\���dx�J���?r��-pgg%�#8W@�#1x��Cr���*�<~}�
-��M�_B����/ezqS������
-�?⪑�
+Salted__g�� ���T{£3liշ�D(X��sp�a�K�p��G�g����`!�F�M�$cp����D�J�{��%�bo�d�02�ѳ=G��H�J��
diff --git a/pipelines/scheduled/launch_upload_jobs.yml.signature b/pipelines/scheduled/launch_upload_jobs.yml.signature
diff --git a/utilities/build_envs.sh b/utilities/build_envs.sh
@@ -138,17 +138,39 @@ S3_BUCKET="${S3_BUCKET:-julialangnightlies}"
 S3_BUCKET_PREFIX="${S3_BUCKET_PREFIX:-bin}"
 
 # We generally upload to multiple upload targets
-UPLOAD_TARGETS=(
+UPLOAD_TARGETS=()
+
+if [[ "${BUILDKITE_BRANCH}" == master ]] || [[ "${BUILDKITE_BRANCH}" == release-* ]] || [[ "${BUILDKITE_TAG}" == v* ]]; then
     # First, we have the canonical fully-specified upload target
-    "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}"
+    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}-${ARCH?}" )
 
     # Next, we have the "majmin/latest" upload target
-    "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}"
-)
+    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/${MAJMIN?}/julia-latest-${OS?}-${ARCH?}" )
 
-# The absolute latest upload target is only for if we're on the `master` branch
-if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
-    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}" )
+    # The absolute latest upload target is only for if we're on the `master` branch
+    if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/${ARCH?}/julia-latest-${OS?}-${ARCH?}" )
+    fi
+
+    # Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64`
+    # and `x86`, and ending in `-linux64` and `-linux32`, although I would very much like to stop doing that.
+    if [[ "${ARCH}" == "x86_64" ]]; then
+        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64" )
+        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64" )
+
+        # Only upload to absolute latest if we're on `master`
+        if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+            UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64" )
+        fi
+    elif [[ "${ARCH}" == "i686" ]]; then
+        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32" )
+        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32" )
+
+        # Only upload to absolute latest if we're on `master`
+        if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+            UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32" )
+        fi
+    fi
 fi
 
 # If we're a pull request build, upload to a special `-prXXXX` location
@@ -157,26 +179,6 @@ if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then
 fi
 
 
-# Finally, for compatibility, we keep on uploading x86_64 and i686 targets to folders called `x64`
-# and `x86`, and ending in `-linux64` and `-linux32`, although I would very much like to stop doing that.
-if [[ "${ARCH}" == "x86_64" ]]; then
-    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}64" )
-    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/${MAJMIN?}/julia-latest-${OS?}64" )
-
-    # Only upload to absolute latest if we're on `master`
-    if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
-        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x64/julia-latest-${OS?}64" )
-    fi
-elif [[ "${ARCH}" == "i686" ]]; then
-    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-${TAR_VERSION?}-${OS?}32" )
-    UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/${MAJMIN?}/julia-latest-${OS?}32" )
-
-    # Only upload to absolute latest if we're on `master`
-    if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
-        UPLOAD_TARGETS+=( "${S3_BUCKET}/${S3_BUCKET_PREFIX}/${OS?}/x86/julia-latest-${OS?}32" )
-    fi
-fi
-
 # We used to name our darwin builds as `julia-*-mac64.tar.gz`, instead of `julia-*-macos-x86_64.tar.gz`.
 # Let's copy things over to the `mac` OS name for backwards compatibility:
 if [[ "${OS?}" == "macos" ]] || [[ "${OS?}" == "windows" ]]; then
diff --git a/utilities/upload_julia.sh b/utilities/upload_julia.sh
@@ -13,91 +13,97 @@ echo "--- Download ${UPLOAD_FILENAME}.tar.gz to ."
 buildkite-agent artifact download "${UPLOAD_FILENAME}.tar.gz" .
 
 # These are the extensions that we will always upload
-UPLOAD_EXTENSIONS=("tar.gz" "tar.gz.asc")
-# If we're on macOS, we need to re-sign the tarball
-if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then
-    echo "--- [mac] Unlock keychain"
-
-    # This _must_ be an absolute path
-    KEYCHAIN_PATH="$(pwd)/.buildkite/secrets/macos_codesigning.keychain"
-    MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362"
-
-    # Add the keychain to the list of keychains to search, then unlock it
-    security -v list-keychains -s -d user "${KEYCHAIN_PATH}"
-    security unlock-keychain -p "keychainpassword" "${KEYCHAIN_PATH}"
-    security find-identity -p codesigning "${KEYCHAIN_PATH}"
-
-    echo "--- [mac] Codesign tarball contents"
-    mkdir -p "${JULIA_INSTALL_DIR}"
-    tar zxf "${UPLOAD_FILENAME}.tar.gz" -C "${JULIA_INSTALL_DIR}" --strip-components 1
-    .buildkite/utilities/macos/codesign.sh \
-        --keychain "${KEYCHAIN_PATH}" \
-        --identity "${MACOS_CODESIGN_IDENTITY}" \
-        "${JULIA_INSTALL_DIR}"
-
-    echo "--- [mac] Update checksums for stdlib cachefiles"
-    ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/macos/update_stdlib_pkgimage_checksums.jl
-
-    # Immediately re-compress that tarball for upload
-    echo "--- [mac] Re-compress codesigned tarball"
-    rm -f "${UPLOAD_FILENAME}.tar.gz"
-    tar zcf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}"
-
-    # Make a `.dmg` out of those files
-    echo "--- [mac] Build .dmg"
-    KEYCHAIN_PATH="${KEYCHAIN_PATH}" MACOS_CODESIGN_IDENTITY="${MACOS_CODESIGN_IDENTITY}" \
-        .buildkite/utilities/macos/build_dmg.sh
-
-    # Add the `.dmg` to our upload targets
-    UPLOAD_EXTENSIONS+=( "dmg" )
-elif [[ "${OS}" == "windows" || "${OS}" == "windowsnogpl" ]]; then
-    echo "--- [windows] Extract pre-built Julia"
-    mkdir -p "${JULIA_INSTALL_DIR}"
-    tar zxf "${UPLOAD_FILENAME}.tar.gz" -C "${JULIA_INSTALL_DIR}" --strip-components 1
-
-    echo "--- [windows] install innosetup"
-    mkdir -p dist-extras
-    curl --fail -L -o 'dist-extras/is.exe' 'https://cache.julialang.org/https://www.jrsoftware.org/download.php/is.exe' || curl --fail -L -o 'dist-extras/is.exe' 'https://www.jrsoftware.org/download.php/is.exe'
-    chmod a+x dist-extras/is.exe
-    MSYS2_ARG_CONV_EXCL='*' ./dist-extras/is.exe \
-        /DIR="$(cygpath -w "$(pwd)/dist-extras/inno")" \
-        /PORTABLE=1 \
-        /CURRENTUSER \
-        /VERYSILENT
-    rm -f dist-extras/is.exe
-
-    echo "--- [windows] make exe"
-    codesign_script="$(pwd)/.buildkite/utilities/windows/codesign.sh"
-    certificate="$(pwd)/.buildkite/secrets/windows_codesigning.pfx"
-    iss_file="$(pwd)/.buildkite/utilities/windows/build-installer.iss"
-    MSYS2_ARG_CONV_EXCL='*' ./dist-extras/inno/iscc.exe \
-        /DAppVersion=${JULIA_VERSION} \
-        /DSourceDir="$(cygpath -w "$(pwd)/${JULIA_INSTALL_DIR}")" \
-        /DRepoDir="$(cygpath -w "$(pwd)")" \
-        /F"${UPLOAD_FILENAME}" \
-        /O"$(cygpath -w "$(pwd)")" \
-        /Dsign=true \
-        /Smysigntool="bash.exe '${codesign_script}' --certificate='${certificate}' \$f" \
-        "$(cygpath -w "${iss_file}")"
-
-    # Add the `.exe` to our upload targets
-    UPLOAD_EXTENSIONS+=( "exe" )
-
-    # Immediately re-compress that tarball for upload
-    echo "--- [windows] Re-compress codesigned tarball"
-    rm -f "${UPLOAD_FILENAME}.tar.gz"
-    tar zcf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}"
-
-    # Use 7z to create a `.zip` file to upload as well
-    echo "--- [windows] make zip"
-    PATH="${JULIA_INSTALL_DIR}/libexec:${JULIA_INSTALL_DIR}/libexec/julia:${PATH}" \
-    7z.exe a "${UPLOAD_FILENAME}.zip" "$(cygpath -w "$(pwd)/${JULIA_INSTALL_DIR}")"
-    UPLOAD_EXTENSIONS+=( "zip" )
+UPLOAD_EXTENSIONS=( "tar.gz" )
+
+# Only codesign if we are not on a pull request build.
+# Pull request builds only upload unsigned tarballs.
+if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+    # If we're on macOS, we need to re-sign the tarball
+    if [[ "${OS}" == "macos" || "${OS}" == "macosnogpl" ]]; then
+        echo "--- [mac] Unlock keychain"
+
+        # This _must_ be an absolute path
+        KEYCHAIN_PATH="$(pwd)/.buildkite/secrets/macos_codesigning.keychain"
+        MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362"
+
+        # Add the keychain to the list of keychains to search, then unlock it
+        security -v list-keychains -s -d user "${KEYCHAIN_PATH}"
+        security unlock-keychain -p "keychainpassword" "${KEYCHAIN_PATH}"
+        security find-identity -p codesigning "${KEYCHAIN_PATH}"
+
+        echo "--- [mac] Codesign tarball contents"
+        mkdir -p "${JULIA_INSTALL_DIR}"
+        tar zxf "${UPLOAD_FILENAME}.tar.gz" -C "${JULIA_INSTALL_DIR}" --strip-components 1
+        .buildkite/utilities/macos/codesign.sh \
+            --keychain "${KEYCHAIN_PATH}" \
+            --identity "${MACOS_CODESIGN_IDENTITY}" \
+            "${JULIA_INSTALL_DIR}"
+
+        echo "--- [mac] Update checksums for stdlib cachefiles"
+        ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/macos/update_stdlib_pkgimage_checksums.jl
+
+        # Immediately re-compress that tarball for upload
+        echo "--- [mac] Re-compress codesigned tarball"
+        rm -f "${UPLOAD_FILENAME}.tar.gz"
+        tar zcf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}"
+
+        # Make a `.dmg` out of those files
+        echo "--- [mac] Build .dmg"
+        KEYCHAIN_PATH="${KEYCHAIN_PATH}" MACOS_CODESIGN_IDENTITY="${MACOS_CODESIGN_IDENTITY}" \
+            .buildkite/utilities/macos/build_dmg.sh
+
+        # Add the `.dmg` to our upload targets
+        UPLOAD_EXTENSIONS+=( "dmg" )
+    elif [[ "${OS}" == "windows" || "${OS}" == "windowsnogpl" ]]; then
+        echo "--- [windows] Extract pre-built Julia"
+        mkdir -p "${JULIA_INSTALL_DIR}"
+        tar zxf "${UPLOAD_FILENAME}.tar.gz" -C "${JULIA_INSTALL_DIR}" --strip-components 1
+
+        echo "--- [windows] install innosetup"
+        mkdir -p dist-extras
+        curl --fail -L -o 'dist-extras/is.exe' 'https://cache.julialang.org/https://www.jrsoftware.org/download.php/is.exe' || curl --fail -L -o 'dist-extras/is.exe' 'https://www.jrsoftware.org/download.php/is.exe'
+        chmod a+x dist-extras/is.exe
+        MSYS2_ARG_CONV_EXCL='*' ./dist-extras/is.exe \
+            /DIR="$(cygpath -w "$(pwd)/dist-extras/inno")" \
+            /PORTABLE=1 \
+            /CURRENTUSER \
+            /VERYSILENT
+        rm -f dist-extras/is.exe
+
+        echo "--- [windows] make exe"
+        codesign_script="$(pwd)/.buildkite/utilities/windows/codesign.sh"
+        certificate="$(pwd)/.buildkite/secrets/windows_codesigning.pfx"
+        iss_file="$(pwd)/.buildkite/utilities/windows/build-installer.iss"
+        MSYS2_ARG_CONV_EXCL='*' ./dist-extras/inno/iscc.exe \
+            /DAppVersion=${JULIA_VERSION} \
+            /DSourceDir="$(cygpath -w "$(pwd)/${JULIA_INSTALL_DIR}")" \
+            /DRepoDir="$(cygpath -w "$(pwd)")" \
+            /F"${UPLOAD_FILENAME}" \
+            /O"$(cygpath -w "$(pwd)")" \
+            /Dsign=true \
+            /Smysigntool="bash.exe '${codesign_script}' --certificate='${certificate}' \$f" \
+            "$(cygpath -w "${iss_file}")"
+
+        # Add the `.exe` to our upload targets
+        UPLOAD_EXTENSIONS+=( "exe" )
+
+        # Immediately re-compress that tarball for upload
+        echo "--- [windows] Re-compress codesigned tarball"
+        rm -f "${UPLOAD_FILENAME}.tar.gz"
+        tar zcf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}"
+
+        # Use 7z to create a `.zip` file to upload as well
+        echo "--- [windows] make zip"
+        PATH="${JULIA_INSTALL_DIR}/libexec:${JULIA_INSTALL_DIR}/libexec/julia:${PATH}" \
+        7z.exe a "${UPLOAD_FILENAME}.zip" "$(cygpath -w "$(pwd)/${JULIA_INSTALL_DIR}")"
+        UPLOAD_EXTENSIONS+=( "zip" )
+    fi
+
+    echo "--- GPG-sign the tarball"
+    .buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "${UPLOAD_FILENAME}.tar.gz"
+    UPLOAD_EXTENSIONS+=( "tar.gz.asc" )
 fi
 
-echo "--- GPG-sign the tarball"
-.buildkite/utilities/sign_tarball.sh .buildkite/secrets/tarball_signing.gpg "${UPLOAD_FILENAME}.tar.gz"
-
 # Helper function to explicitly `wait` on each given PID.
 # Because `wait` returns the exit code of the waited-upon PID,
 # this (in combination with `set -e` above) ends execution if
@@ -109,7 +115,7 @@ wait_pids() {
 }
 
 # First, upload our signed products to buildkite, for easy downloading
-echo "--- Upload signed products to buildkite"
+echo "--- Upload products to buildkite"
 PIDS=()
 for EXT in "${UPLOAD_EXTENSIONS[@]}"; do
     buildkite-agent artifact upload "${UPLOAD_FILENAME}.${EXT}" &

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		`-Salted__��*��8��ũ.O��ʶ��S��I�"��X.Q٘oKS��(� ��T��w&��k?_s��_I�%��ߙ;~0�`
	`1`	`+Salted__V:l*SY'��rc�\b��9�T�nX ��vUd"�w�~0B��0��&��7�^`
	`2`	`+\|IKg�v<;T�M��H��`
-Original file line number
+Diff line change
@@ @@ -1,3 +1 @@ @@
 -Salted__�R��e��\���dx�J���?r��-pgg%�#8W@�#1x��Cr���*�<~}�
 -��M�_B����/ezqS������
 -�?⪑�
 +Salted__g�� ���T{£3liշ�D(X��sp�a�K�p��G�g����`!�F�M�$cp����D�J�{��%�bo�d�02�ѳ=G��H�J��