[anza migration] replace binaries backend (#6)

yihau · willhickey · commit b84d81b7a51c · 2024-03-05T09:43:25.000-06:00
* ci: add upload-gcs-artifact

* ci: publish release binaries to GCS

* ci: redirect github repo to anza-xyz

* ci: publish windows binaries to GCS

* replace release.solana.com with release.anza.xyz

* use a explicit name for credential
diff --git a/.github/workflows/release-artifacts-auto.yml b/.github/workflows/release-artifacts-auto.yml
@@ -14,14 +14,12 @@ concurrency:
 
 jobs:
   release-artifacts:
-    if: github.repository == 'solana-labs/solana'
+    if: github.repository == 'anza-xyz/agave'
     uses: ./.github/workflows/release-artifacts.yml
     with:
       commit: ${{ github.sha }}
     secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}
 
   error_reporting:
     needs:
diff --git a/.github/workflows/release-artifacts-manually.yml b/.github/workflows/release-artifacts-manually.yml
@@ -14,6 +14,4 @@ jobs:
     with:
       commit: ${{ github.event.inputs.commit }}
     secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}
diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml
@@ -7,11 +7,7 @@ on:
         required: false
         type: string
     secrets:
-      AWS_ACCESS_KEY_ID:
-        required: true
-      AWS_SECRET_ACCESS_KEY:
-        required: true
-      AWS_S3_BUCKET:
+      GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:
         required: true
 
 jobs:
@@ -71,17 +67,17 @@ jobs:
         shell: bash
         run: |
           FOLDER_NAME=${{ steps.build.outputs.tag || steps.build.outputs.channel }}
-          mkdir -p "github-action-s3-upload/$FOLDER_NAME"
-          cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "github-action-s3-upload/$FOLDER_NAME/"
-          cp -v "solana-release-x86_64-pc-windows-msvc.yml" "github-action-s3-upload/$FOLDER_NAME/"
-          cp -v "solana-install-init-x86_64-pc-windows-msvc"* "github-action-s3-upload/$FOLDER_NAME"
+          mkdir -p "windows-release/$FOLDER_NAME"
+          cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/"
+          cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/"
+          cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME"
 
       - name: Upload Artifacts
         if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }}
         uses: actions/upload-artifact@v3
         with:
           name: windows-artifact
-          path: github-action-s3-upload/
+          path: windows-release/
 
   windows-s3-upload:
     if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }}
@@ -92,7 +88,16 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: windows-artifact
-          path: ./github-action-s3-upload
+          path: .windows-release/
+
+      - name: Setup crediential
+        uses: "google-github-actions/auth@v2"
+        with:
+          credentials_json: "${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}"
+
+      - name: Upload files to GCS
+        run: |
+          gcloud storage cp --recursive windows-release/* gs://anza-release/
 
       - name: Upload
         uses: jakejarvis/s3-sync-action@master
@@ -103,7 +108,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
           AWS_REGION: "us-west-1"
-          SOURCE_DIR: "github-action-s3-upload"
+          SOURCE_DIR: "windows-release"
 
   windows-gh-release:
     if: ${{ needs.windows-build.outputs.tag != '' }}
@@ -114,12 +119,12 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: windows-artifact
-          path: ./github-action-s3-upload
+          path: .windows-release/
 
       - name: Release
         uses: softprops/action-gh-release@v1
         with:
           tag_name: ${{ needs.windows-build.outputs.tag }}
           draft: true
           files: |
-            github-action-s3-upload/${{ needs.windows-build.outputs.tag }}/*
+            windows-release/${{ needs.windows-build.outputs.tag }}/*
diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh
@@ -26,14 +26,14 @@ fi
 # upload install script
 source ci/upload-ci-artifact.sh
 
-cat >release.solana.com-install <<EOF
+cat >release.anza.xyz-install <<EOF
 SOLANA_RELEASE=$CHANNEL_OR_TAG
 SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG
-SOLANA_DOWNLOAD_ROOT=https://release.solana.com
+SOLANA_DOWNLOAD_ROOT=https://release.anza.xyz
 EOF
-cat install/solana-install-init.sh >>release.solana.com-install
+cat install/solana-install-init.sh >>release.anza.xyz-install
 
-echo --- AWS S3 Store: "install"
-upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install"
+echo --- GCS: "install"
+upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install"
 echo Published to:
-ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install
+ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/install
diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh
@@ -118,11 +118,11 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
   fi
 
   if [[ -n $BUILDKITE ]]; then
-    echo --- AWS S3 Store: "$file"
-    upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
+    echo --- GCS Store: "$file"
+    upload-gcs-artifact "/solana/$file" gs://anza-release/"$CHANNEL_OR_TAG"/"$file"
 
     echo Published to:
-    $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
+    $DRYRUN ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/"$file"
 
     if [[ -n $TAG ]]; then
       ci/upload-github-release-asset.sh "$file"
diff --git a/ci/upload-ci-artifact.sh b/ci/upload-ci-artifact.sh
@@ -40,3 +40,13 @@ upload-s3-artifact() {
     docker run "${args[@]}"
   )
 }
+
+upload-gcs-artifact() {
+  echo "--- artifact: $1 to $2"
+  docker run --rm \
+    -v "$GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:/application_default_credentials.json" \
+    -v "$PWD:/solana" \
+    -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=/application_default_credentials.json \
+    gcr.io/google.com/cloudsdktool/google-cloud-cli:latest \
+    gcloud storage cp "$1" "$2"
+}
diff --git a/ci/upload-github-release-asset.sh b/ci/upload-github-release-asset.sh
@@ -26,7 +26,7 @@ fi
 # Force CI_REPO_SLUG since sometimes
 # BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the
 # artifact upload to fail
-CI_REPO_SLUG=solana-labs/solana
+CI_REPO_SLUG=anza-xyz/agave
 #if [[ -z $CI_REPO_SLUG ]]; then
 #  echo Error: CI_REPO_SLUG not defined
 #  exit 1
diff --git a/install/solana-install-init.sh b/install/solana-install-init.sh
@@ -16,9 +16,9 @@
 { # this ensures the entire script is downloaded #
 
 if [ -z "$SOLANA_DOWNLOAD_ROOT" ]; then
-    SOLANA_DOWNLOAD_ROOT="https://github.com/solana-labs/solana/releases/download/"
+    SOLANA_DOWNLOAD_ROOT="https://github.com/anza-xyz/agave/releases/download/"
 fi
-GH_LATEST_RELEASE="https://api.github.com/repos/solana-labs/solana/releases/latest"
+GH_LATEST_RELEASE="https://api.github.com/repos/anza-xyz/agave/releases/latest"
 
 set -e
 
diff --git a/install/src/command.rs b/install/src/command.rs
@@ -572,23 +572,23 @@ pub fn init(
 
 fn github_release_download_url(release_semver: &str) -> String {
     format!(
-        "https://github.com/solana-labs/solana/releases/download/v{}/solana-release-{}.tar.bz2",
+        "https://github.com/anza-xyz/agave/releases/download/v{}/solana-release-{}.tar.bz2",
         release_semver,
         crate::build_env::TARGET
     )
 }
 
 fn release_channel_download_url(release_channel: &str) -> String {
     format!(
-        "https://release.solana.com/{}/solana-release-{}.tar.bz2",
+        "https://release.anza.xyz/{}/solana-release-{}.tar.bz2",
         release_channel,
         crate::build_env::TARGET
     )
 }
 
 fn release_channel_version_url(release_channel: &str) -> String {
     format!(
-        "https://release.solana.com/{}/solana-release-{}.yml",
+        "https://release.anza.xyz/{}/solana-release-{}.yml",
         release_channel,
         crate::build_env::TARGET
     )
@@ -905,7 +905,7 @@ fn check_for_newer_github_release(
 
     while page == 1 || releases.len() == PER_PAGE {
         let url = reqwest::Url::parse_with_params(
-            "https://api.github.com/repos/solana-labs/solana/releases",
+            "https://api.github.com/repos/anza-xyz/agave/releases",
             &[
                 ("per_page", &format!("{PER_PAGE}")),
                 ("page", &format!("{page}")),

Original file line number	Diff line number	Diff line change
`@@ -40,3 +40,13 @@ upload-s3-artifact() {`
`40`	`40`	`docker run "${args[@]}"`
`41`	`41`	`)`
`42`	`42`	`}`
	`43`	`+`
	`44`	`+upload-gcs-artifact() {`
	`45`	`+ echo "--- artifact: $1 to $2"`
	`46`	`+ docker run --rm \`
	`47`	`+ -v "$GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:/application_default_credentials.json" \`
	`48`	`+ -v "$PWD:/solana" \`
	`49`	`+ -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=/application_default_credentials.json \`
	`50`	`+ gcr.io/google.com/cloudsdktool/google-cloud-cli:latest \`
	`51`	`+ gcloud storage cp "$1" "$2"`
	`52`	`+}`