feat: upload download.sh to s3 and auto release

Signed-off-by: Bird <[email protected]>

Author: aFlyBird0

.github/workflows/automated-release.yml

@@ -77,6 +77,8 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ap-southeast-1
+      - name: update dtm version on s3
+        run: ./hack/release/update_dtm_version_on_s3.sh ${{ env.TAG }}
       - name: upload dtm core to github release and s3
         run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
       - name: upload plugins to s3

.github/workflows/update-download-script.yml

@@ -0,0 +1,23 @@
+name: update download.sh
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - hack/install.sh
+  workflow_dispatch:
+
+jobs:
+    update-download-script:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+        - name: Configure AWS credentials
+          uses: aws-actions/configure-aws-credentials@v1
+          with:
+            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            aws-region: ap-southeast-1
+        - name: Update download.sh
+          run: aws s3 cp hack/install/download.sh s3://download.devstream.io/download.sh --acl public-read

hack/install/download.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 function init() {
+  # todo customize dtm version, default is latest
   if [ "$(uname)" == "Darwin" ];then
     HOST_OS="darwin"
   elif [ "$(uname)" == "Linux" ];then
@@ -23,23 +24,24 @@ function init() {
 }
 
 function getLatestReleaseVersion() {
-  if [ -n "${GITHUB_TOKEN}" ]; then
-    AUTH_HEADER="-H Authorization: token ${GITHUB_TOKEN}"
-  fi
+  # get latest release version from aws s3
+  STORAGE_BASE_URL=https://download.devstream.io
+  LATEST_VERSION_FILE="latest_version"
+
+  LATEST_VERSION=$(curl -fsSL ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE})
 
-  # like "v1.2.3"
-  latestVersion=$(curl ${AUTH_HEADER} -s https://api.github.com/repos/devstream-io/devstream/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-  if [ -z "$latestVersion" ]; then
+  if [ -z "${LATEST_VERSION}" ];then
     echo "Failed to get latest release version"
     exit 1
   fi
-  echo "Latest dtm release version: ${latestVersion}"
+
+  echo "Got latest release version: ${LATEST_VERSION}"
 }
 
 function downloadDtm() {
   # 1. download the release and rename it to "dtm"
   # 2. count the download count of the release
-  fullReleaseUrl="https://devstream.gateway.scarf.sh/releases/$latestVersion/dtm-$HOST_OS-$HOST_ARCH"
+  fullReleaseUrl="${STORAGE_BASE_URL}/${LATEST_VERSION}/dtm-${HOST_OS}-${HOST_ARCH}"
   echo "Downloading dtm from: $fullReleaseUrl"
   # use -L to follow redirects
   curl -L -o dtm $fullReleaseUrl

hack/release/auto-release-darwin-arm64.sh

@@ -63,7 +63,7 @@ echo "Uploading ${DTM_CORE_BINARY} ..."
 # upload dtm to github release
 github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY}
 # upload dtm to aws s3
-aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY}
+aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read
 echo "${DTM_CORE_BINARY} uploaded."
 
 # upload plugins and .md5 files

hack/release/update_download_sh.sh

@@ -0,0 +1,9 @@
+#! /bin/bash -e
+
+DOWNLOAD_SCRIPT_PATH="hack/install/download.sh"
+STORAGE_BASE_URL=s3://download.devstream.io
+DOWNLOAD_SCRIPT_S3_URL=${STORAGE_BASE_URL}/${DOWNLOAD_SCRIPT_PATH}
+
+# upload download.sh to aws s3
+echo "Uploading ${DOWNLOAD_SCRIPT_PATH} to ${DOWNLOAD_SCRIPT_S3_URL} ..."
+aws s3 cp ${DOWNLOAD_SCRIPT_PATH} ${DOWNLOAD_SCRIPT_S3_URL} --acl public-read

hack/release/update_dtm_version_on_s3.sh

@@ -0,0 +1,14 @@
+#! /bin/bash -e
+
+version=$1
+
+LATEST_VERSION_FILE="latest_version"
+STORAGE_BASE_URL=s3://download.devstream.io
+
+# create latest_version file
+echo "Saving latest version(${version}) to ${LATEST_VERSION_FILE} ..."
+echo $version > ${LATEST_VERSION_FILE}
+
+# create or update latest_version.txt on s3
+aws s3 cp ${LATEST_VERSION_FILE} ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE} --acl public-read
+echo "${LATEST_VERSION_FILE} uploaded."

hack/release/upload_dtm_core.sh

@@ -18,6 +18,6 @@ echo "Uploading ${DTM_CORE_BINARY} ..."
 # upload dtm to github release
 github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY}
 # upload dtm to aws s3
-aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY}
+aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read
 echo "${DTM_CORE_BINARY} uploaded."