Skip to content

Commit 2a1b948

Browse files
AlinsRanAlinsRan
authored
chore(ci): replace coscli with coscmd (#324)
1 parent 740ec67 commit 2a1b948

File tree

4 files changed

+86
-69
lines changed

4 files changed

+86
-69
lines changed

‎.github/workflows/publish-deb.yml‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Build and Publish Package
22

33
on:
4-
create
4+
create:
55

66
jobs:
77
publish_apisix:
@@ -10,6 +10,8 @@ jobs:
1010
timeout-minutes: 60
1111
env:
1212
VAR_DEB_WORKBENCH_DIR: /tmp/output
13+
VAR_COS_BUCKET_REPO: ${{ secrets.VAR_COS_BUCKET_REPO }}
14+
VAR_COS_BUCKET_CI: ${{ secrets.VAR_COS_BUCKET_CI }}
1315
VAR_OS: debian
1416
VAR_CODENAME: bullseye
1517
VAR_OS_RELEASE: bullseye-slim
@@ -73,3 +75,40 @@ jobs:
7375
name: "apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
7476
path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
7577

78+
- name: DEB repo deps init
79+
env:
80+
TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }}
81+
TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }}
82+
run: |
83+
sudo pip install coscmd
84+
sudo -E ./utils/publish-deb.sh init_cos_utils
85+
86+
- name: DEB repo init for debian
87+
run: |
88+
sudo -E ./utils/publish-deb.sh repo_clone
89+
90+
- name: DEB repo package update
91+
env:
92+
DEB_GPG_MAIL: ${{ secrets.DEB_GPG_MAIL }}
93+
GPG_NAME: ${{ secrets.GPG_NAME }}
94+
GPG_MAIL: ${{ secrets.GPG_MAIL }}
95+
run: |
96+
echo "${{ secrets.DEB_GPG_PRIV_KEY }}" >> /tmp/deb-gpg-publish.private
97+
echo "${{ secrets.DEB_GPG_PASSPHRASE }}" >> /tmp/deb-gpg-publish.passphrase
98+
sudo -E ./utils/publish-deb.sh init_freight_utils
99+
sudo -E ./utils/publish-deb.sh init_gpg
100+
101+
- name: DEB repo backup for debian
102+
run: |
103+
sudo -E ./utils/publish-deb.sh dists_backup
104+
105+
- name: DEB repo refresh for debian
106+
run: |
107+
sudo -E ./utils/publish-deb.sh repo_rebuild
108+
sudo -E ./utils/publish-deb.sh repo_ci_upload
109+
110+
- name: DEB repo publish for debian
111+
run: |
112+
sudo -E ./utils/publish-deb.sh repo_upload
113+
sudo -E ./utils/publish-rpm.sh repo_publish
114+

‎.github/workflows/publish.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ jobs:
130130
TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }}
131131
TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }}
132132
run: |
133+
sudo pip install coscmd
133134
sudo -E ./utils/publish-rpm.sh init_cos_utils
134135
135136
- name: RPM repo init for centos and redhat

‎utils/publish-deb.sh‎

Lines changed: 26 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@ env
1010
# Runtime default config
1111
# =======================================
1212
VAR_FREIGHT_UTILS_VERSION=${VAR_FREIGHT_UTILS_VERSION:-v0.3.13}
13-
VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta}
1413
VAR_DEB_WORKBENCH_DIR=${VAR_DEB_WORKBENCH_DIR:-/tmp/output}
1514
VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/deb-gpg-publish.private}
1615
VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/deb-gpg-publish.passphrase}
1716

18-
COS_CMD=coscli
17+
COS_REGION=${COS_REGION:-"ap-guangzhou"}
18+
COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"}
19+
COS_PART_SIZE=${COS_PART_SIZE:-"10"}
20+
VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com"
21+
VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com"
22+
1923
TAG_DATE=$(date +%Y%m%d)
2024
ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`}
2125
arch_path=""
@@ -41,30 +45,11 @@ _EOC_
4145
# =======================================
4246
# COS extension
4347
# =======================================
44-
func_cos_utils_install() {
45-
if [[ $ARCH == "arm64" ]] || [[ $ARCH == "aarch64" ]]; then
46-
wget https://github.com/tencentyun/coscli/archive/refs/tags/${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz
47-
tar -zxvf ${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz
48-
cd coscli-* && go build
49-
mv coscli ../
50-
else
51-
sudo curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${VAR_TENCENT_COS_UTILS_VERSION}/coscli-linux"
52-
sudo chmod 755 /usr/bin/coscli
53-
fi
54-
}
55-
5648
func_cos_utils_credential_init() {
57-
# ${1} - COS endpoint
58-
# ${2} - COS SECRET_ID
59-
# ${3} - COS SECRET_KEY
60-
cat > "${HOME}/.cos.yaml" <<_EOC_
61-
cos:
62-
base:
63-
secretid: ${2}
64-
secretkey: ${3}
65-
sessiontoken: ""
66-
protocol: https
67-
_EOC_
49+
# ${1} - COS SECRET_ID
50+
# ${2} - COS SECRET_KEY
51+
# ${3} - COS bucket name
52+
coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE}
6853
}
6954

7055

@@ -91,21 +76,20 @@ func_dists_backup() {
9176
# ${1} - bucket name
9277
# ${2} - COS path
9378
# ${3} - backup tag
94-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/dists" "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true
79+
coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${2}/dists" "/packages/${arch_path}backup/${2}_dists_${3}"
9580
}
9681

9782
func_pool_clone() {
9883
# ${1} - bucket name
9984
# ${2} - COS path
10085
# ${3} - local pool path
101-
10286
mkdir -p ${3}
10387
# --part-size indicates the file chunk size.
10488
# when the file is larger than --part-size, coscli will chunk the file by --part-size.
10589
# when uploading/downloading the file in chunks, it will enable breakpoint transfer by default,
10690
# which will generate cosresumabletask file and interfere with the file integrity.
10791
# ref: https://cloud.tencent.com/document/product/436/63669
108-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/pool" "${3}"
92+
coscmd -b "${1}" -r "${COS_GLOBAL_REGION}" download -r "/packages/${arch_path}${2}/pool" "${3}"
10993
}
11094

11195
func_dists_rebuild() {
@@ -139,8 +123,8 @@ func_dists_rebuild() {
139123
}
140124

141125
func_dists_upload_ci_repo() {
142-
$COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}" || true
143-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "${1}" "cos://${2}/packages/${arch_path}${3}/dists/"
126+
coscmd -b "${2}" delete -r -f "/packages/${arch_path}${3}" || true
127+
coscmd -b "${2}" -r "${COS_GLOBAL_REGION}" upload -r "${1}" "/packages/${arch_path}${3}/dists"
144128
}
145129

146130
func_deb_upload() {
@@ -152,33 +136,34 @@ func_deb_upload() {
152136
# We will only upload apisix and apisix-base,
153137
# so the directory is fixed: pool/main/a.
154138
# Regardless of other packages.
155-
156-
export COS_CMD=$COS_CMD
157139
export arch_path=$arch_path
158140
export BUCKET=$2
159141
export OS=$3
160142
export CODENAME=$4
161143
find "${1}" -type f -name "apisix_*.deb" \
162144
-exec echo "upload : {}" \; \
163145
-exec sh -c 'file=$(basename {}); \
164-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \;
146+
coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \;
165147

166148
find "${1}" -type f -name "apisix-base*.deb" \
167149
-exec echo "upload : {}" \; \
168150
-exec sh -c 'file=$(basename {}); \
169-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \;
151+
coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \;
170152
}
171153

172154
func_repo_publish() {
173-
$COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}/dists" || true
174-
$COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "cos://${1}/packages/${arch_path}${3}/dists" "cos://${2}/packages/${arch_path}${3}/dists"
155+
# ${1} - CI bucket
156+
# ${2} - repo publish bucket
157+
# ${3} - COS path
158+
coscmd delete -r -f "/packages/${arch_path}${3}/dists" || true
159+
coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${3}/dists" "/packages/${arch_path}${3}/dists"
175160
}
176161

177162
func_repo_backup_remove() {
178163
# ${1} - bucket name
179164
# ${2} - COS path
180165
# ${3} - backup tag
181-
$COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true
166+
coscmd -b "${1}" delete -r -f "/packages/${arch_path}backup/${2}_dists_${3}" || true
182167
}
183168

184169
# =======================================
@@ -188,8 +173,7 @@ case_opt=$1
188173

189174
case ${case_opt} in
190175
init_cos_utils)
191-
func_cos_utils_install
192-
func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}"
176+
func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}"
193177
;;
194178
init_freight_utils)
195179
func_freight_utils_install
@@ -210,8 +194,10 @@ repo_clone)
210194
repo_rebuild)
211195
func_dists_rebuild "/tmp/old_pool" "/tmp/freight" ${VAR_DEB_WORKBENCH_DIR} ${VAR_CODENAME}
212196
;;
213-
repo_upload)
197+
repo_ci_upload)
214198
func_dists_upload_ci_repo "/tmp/freight/cache/dists" "${VAR_COS_BUCKET_CI}" "${VAR_OS}"
199+
;;
200+
repo_upload)
215201
func_deb_upload "${VAR_DEB_WORKBENCH_DIR}" "${VAR_COS_BUCKET_REPO}" "${VAR_OS}" "${VAR_CODENAME}"
216202
;;
217203
repo_publish)

‎utils/publish-rpm.sh‎

Lines changed: 19 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,17 @@ env
99
# =======================================
1010
# Runtime default config
1111
# =======================================
12-
VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta}
1312
VAR_RPM_WORKBENCH_DIR=${VAR_RPM_WORKBENCH_DIR:-/tmp/output}
1413
VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/rpm-gpg-publish.private}
1514
VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/rpm-gpg-publish.passphrase}
1615
ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`}
1716

17+
COS_REGION=${COS_REGION:-"ap-guangzhou"}
18+
COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"}
19+
COS_PART_SIZE=${COS_PART_SIZE:-"10"}
20+
VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com"
21+
VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com"
22+
1823
# =======================================
1924
# GPG extension
2025
# =======================================
@@ -42,24 +47,11 @@ func_gpg_key_load() {
4247
# =======================================
4348
# COS extension
4449
# =======================================
45-
func_cos_utils_install() {
46-
# ${1} - COS util version
47-
curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${1}/coscli-linux"
48-
chmod 755 /usr/bin/coscli
49-
}
50-
5150
func_cos_utils_credential_init() {
52-
# ${1} - COS endpoint
53-
# ${2} - COS SECRET_ID
54-
# ${3} - COS SECRET_KEY
55-
cat > "${HOME}/.cos.yaml" <<_EOC_
56-
cos:
57-
base:
58-
secretid: ${2}
59-
secretkey: ${3}
60-
sessiontoken: ""
61-
protocol: https
62-
_EOC_
51+
# ${1} - COS SECRET_ID
52+
# ${2} - COS SECRET_KEY
53+
# ${3} - COS bucket name
54+
coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE}
6355
}
6456

6557
# =======================================
@@ -77,25 +69,25 @@ func_repo_clone() {
7769
# ${3} - target path
7870

7971
# --part-size indicates the file chunk size.
80-
# when the file is larger than --part-size, coscli will chunk the file by --part-size.
72+
# when the file is larger than --part-size, coscmd will chunk the file by --part-size.
8173
# when uploading/downloading the file in chunks, it will enable breakpoint transfer by default,
8274
# which will generate cosresumabletask file and interfere with the file integrity.
8375
# ref: https://cloud.tencent.com/document/product/436/63669
84-
coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "${3}"
76+
coscmd -b "${1}" -r "${COS_GLOBAL_REGION}" download -r "/packages/${2}" "${3}"
8577
}
8678

8779
func_repo_backup() {
8880
# ${1} - bucket name
8981
# ${2} - COS path
9082
# ${3} - backup tag
91-
coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "cos://${1}/packages/backup/${2}_${3}"
83+
coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${2}" "/packages/backup/${2}_${3}"
9284
}
9385

9486
func_repo_backup_remove() {
9587
# ${1} - bucket name
9688
# ${2} - COS path
9789
# ${3} - backup tag
98-
coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/backup/${2}_${3}"
90+
coscmd -b "${1}" delete -r -f "/packages/backup/${2}_${3}"
9991
}
10092

10193
func_repo_repodata_rebuild() {
@@ -117,16 +109,16 @@ func_repo_upload() {
117109
# ${1} - local path
118110
# ${2} - bucket name
119111
# ${3} - COS path
120-
coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true
121-
coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "${1}" "cos://${2}/packages/${3}"
112+
coscmd -b "${2}" delete -r -f "/packages/${3}" || true
113+
coscmd -b "${2}" -r ${COS_GLOBAL_REGION} upload -r "${1}" "/packages/${3}"
122114
}
123115

124116
func_repo_publish() {
125117
# ${1} - CI bucket
126118
# ${2} - repo publish bucket
127119
# ${3} - COS path
128-
coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true
129-
coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${3}" "cos://${2}/packages/${3}"
120+
coscmd delete -r -f "/packages/${3}" || true
121+
coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${3}" "packages/${3}"
130122
}
131123

132124
# =======================================
@@ -135,8 +127,7 @@ func_repo_publish() {
135127
case_opt=$1
136128
case ${case_opt} in
137129
init_cos_utils)
138-
func_cos_utils_install "${VAR_TENCENT_COS_UTILS_VERSION}"
139-
func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}"
130+
func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}"
140131
;;
141132
repo_init)
142133
# create basic repo directory structure

0 commit comments

Comments
 (0)