Skip to content

Commit 6e15b53

Browse files
aFlyBird0aFlyBird0
authored
Merge pull request #1301 from aFlyBird0/feat-upload-dtm-to-s3
feat: Use aws s3 to download dtm core
2 parents f07037f + 5162e08 commit 6e15b53

File tree

8 files changed

+97
-32
lines changed

8 files changed

+97
-32
lines changed

‎.github/workflows/automated-release.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
4343
aws-region: ap-southeast-1
4444
- name: upload core
45-
run: bash -e ./hack/release/upload_assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
45+
run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
4646
- name: upload plugin
4747
run: aws s3 cp ~/.devstream/plugins/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read
4848

@@ -77,8 +77,10 @@ jobs:
7777
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
7878
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
7979
aws-region: ap-southeast-1
80-
- name: upload
81-
run: bash -e ./hack/release/upload_assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
82-
- name: upload plugin
80+
- name: update dtm version on s3
81+
run: ./hack/release/update_dtm_version_on_s3.sh ${{ env.TAG }}
82+
- name: upload dtm core to github release and s3
83+
run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
84+
- name: upload plugins to s3
8385
run: aws s3 cp ~/.devstream/plugins/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read
8486

‎.github/workflows/update-download-script.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: update download.sh
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
paths:
7+
- hack/install.sh
8+
workflow_dispatch:
9+
10+
jobs:
11+
update-download-script:
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v3
16+
- name: Configure AWS credentials
17+
uses: aws-actions/configure-aws-credentials@v1
18+
with:
19+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
20+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
21+
aws-region: ap-southeast-1
22+
- name: Update download.sh
23+
run: aws s3 cp hack/install/download.sh s3://download.devstream.io/download.sh --acl public-read

‎hack/install/download.sh‎

100644100755
Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
22

33
function init() {
4+
# todo customize dtm version, default is latest
45
if [ "$(uname)" == "Darwin" ];then
56
HOST_OS="darwin"
67
elif [ "$(uname)" == "Linux" ];then
@@ -23,23 +24,24 @@ function init() {
2324
}
2425

2526
function getLatestReleaseVersion() {
26-
if [ -n "${GITHUB_TOKEN}" ]; then
27-
AUTH_HEADER="-H Authorization: token ${GITHUB_TOKEN}"
28-
fi
27+
# get latest release version from aws s3
28+
STORAGE_BASE_URL=https://download.devstream.io
29+
LATEST_VERSION_FILE="latest_version"
30+
31+
LATEST_VERSION=$(curl -fsSL ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE})
2932

30-
# like "v1.2.3"
31-
latestVersion=$(curl ${AUTH_HEADER} -s https://api.github.com/repos/devstream-io/devstream/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
32-
if [ -z "$latestVersion" ]; then
33+
if [ -z "${LATEST_VERSION}" ];then
3334
echo "Failed to get latest release version"
3435
exit 1
3536
fi
36-
echo "Latest dtm release version: ${latestVersion}"
37+
38+
echo "Got latest release version: ${LATEST_VERSION}"
3739
}
3840

3941
function downloadDtm() {
4042
# 1. download the release and rename it to "dtm"
4143
# 2. count the download count of the release
42-
fullReleaseUrl="https://devstream.gateway.scarf.sh/releases/$latestVersion/dtm-$HOST_OS-$HOST_ARCH"
44+
fullReleaseUrl="${STORAGE_BASE_URL}/${LATEST_VERSION}/dtm-${HOST_OS}-${HOST_ARCH}"
4345
echo "Downloading dtm from: $fullReleaseUrl"
4446
# use -L to follow redirects
4547
curl -L -o dtm $fullReleaseUrl

‎hack/release/auto-release-darwin-arm64.sh‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ plugin_dir=~/.devstream/plugins
3232

3333
GOOS=$(go env GOOS)
3434
GOARCH=$(go env GOARCH)
35+
DTM_CORE_BINARY=dtm-${GOOS}-${GOARCH}
36+
STORAGE_BASE_URL=s3://download.devstream.io
37+
STORAGE_URL_WITH_TAG=${STORAGE_BASE_URL}/${tag}
3538

3639
if [ ! $tag ] || [ ! $user ] || [ ! $repo ] || [ ! $github_token ] || [ ! $plugin_dir ]; then
3740
echo "The following variables cannot be empty!"
@@ -56,11 +59,15 @@ make build -j8
5659
go install github.com/github-release/github-release@latest
5760

5861
# upload dtm
59-
echo 'Uploading 'dtm-${GOOS}-${GOARCH}' ...'
60-
github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name dtm-${GOOS}-${GOARCH}
61-
echo dtm-${GOOS}-${GOARCH}' uploaded.'
62+
echo "Uploading ${DTM_CORE_BINARY} ..."
63+
# upload dtm to github release
64+
github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY}
65+
# upload dtm to aws s3
66+
aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read
67+
echo "${DTM_CORE_BINARY} uploaded."
6268

6369
# upload plugins and .md5 files
64-
# In order to upload plug-ins to s3, you need to download awscli. After downloading awscli, you need to configure aws credentials.
70+
# In order to upload plug-ins to s3, you need to download aws cli.
71+
# After downloading aws cli, you need to configure aws credentials.
6572
pip3 install awscli
66-
aws s3 cp $plugin_dir s3://download.devstream.io/${tag} --recursive --acl public-read
73+
aws s3 cp $plugin_dir $STORAGE_URL_WITH_TAG --recursive --acl public-read

‎hack/release/update_download_sh.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#! /bin/bash -e
2+
3+
DOWNLOAD_SCRIPT_PATH="hack/install/download.sh"
4+
STORAGE_BASE_URL=s3://download.devstream.io
5+
DOWNLOAD_SCRIPT_S3_URL=${STORAGE_BASE_URL}/${DOWNLOAD_SCRIPT_PATH}
6+
7+
# upload download.sh to aws s3
8+
echo "Uploading ${DOWNLOAD_SCRIPT_PATH} to ${DOWNLOAD_SCRIPT_S3_URL} ..."
9+
aws s3 cp ${DOWNLOAD_SCRIPT_PATH} ${DOWNLOAD_SCRIPT_S3_URL} --acl public-read

‎hack/release/update_dtm_version_on_s3.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#! /bin/bash -e
2+
3+
version=$1
4+
5+
LATEST_VERSION_FILE="latest_version"
6+
STORAGE_BASE_URL=s3://download.devstream.io
7+
8+
# create latest_version file
9+
echo "Saving latest version(${version}) to ${LATEST_VERSION_FILE} ..."
10+
echo $version > ${LATEST_VERSION_FILE}
11+
12+
# create or update latest_version.txt on s3
13+
aws s3 cp ${LATEST_VERSION_FILE} ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE} --acl public-read
14+
echo "${LATEST_VERSION_FILE} uploaded."

‎hack/release/upload_assets.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎hack/release/upload_dtm_core.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#! /bin/bash -e
2+
3+
github_token=$1
4+
tag=$2
5+
GOOS=$3
6+
GOARCH=$4
7+
8+
DTM_CORE_BINARY=dtm-${GOOS}-${GOARCH}
9+
STORAGE_BASE_URL=s3://download.devstream.io
10+
STORAGE_URL_WITH_TAG=${STORAGE_BASE_URL}/${tag}
11+
12+
user=devstream-io
13+
repo=devstream
14+
plugin_dir=~/.devstream/plugins
15+
16+
# upload dtm core
17+
echo "Uploading ${DTM_CORE_BINARY} ..."
18+
# upload dtm to github release
19+
github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY}
20+
# upload dtm to aws s3
21+
aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read
22+
echo "${DTM_CORE_BINARY} uploaded."
23+

0 commit comments

Comments
 (0)