feat: support build debian repo (#221)

soulbird · web-flow · commit 93dd736620e6 · 2022-07-08T15:34:25.000+08:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,7 +1,31 @@
 # Use the latest 2.1 version of CircleCI pipeline process engine.
 # See: https://circleci.com/docs/2.0/configuration-reference
 version: 2.1
-
+commands:
+  description: publish deb package
+  publish_package:
+    steps:
+      - run:
+          name: backup dists to ci bucket
+          command: |
+            sudo -E ./utils/publish-deb.sh dists_backup
+      - run:
+          name: download pool dir to local
+          command: |
+            sudo -E ./utils/publish-deb.sh repo_clone
+      - run:
+          name: rebuild deb repo
+          command: |
+            sudo -E ./utils/publish-deb.sh repo_rebuild
+      - run:
+          name: upload repo
+          command: |
+            sudo -E ./utils/publish-deb.sh repo_upload
+      - run:
+          name: pulish repo
+          command: |
+            sudo -E ./utils/publish-deb.sh repo_publish
+            sudo -E ./utils/publish-deb.sh repo_backup_remove
 # Define a job to be invoked later in a workflow.
 # See: https://circleci.com/docs/2.0/configuration-reference/#jobs
 jobs:
@@ -118,6 +142,7 @@ jobs:
     environment:
       - VAR_DEB_WORKBENCH_DIR: /tmp/output
       - VAR_OS: debian
+      - VAR_CODENAME: bullseye
       - VAR_OS_RELEASE: bullseye-slim
     
     steps:
@@ -126,17 +151,14 @@ jobs:
           name: Init basic publish env
           command: |
             mkdir -p "${VAR_DEB_WORKBENCH_DIR}"
-            wget https://github.com/tencentyun/coscli/archive/refs/tags/v0.11.0-beta.tar.gz
-            tar -zxvf v0.11.0-beta.tar.gz
-            cd coscli-0.11.0-beta && go build
-            sudo mv coscli ../
-            cat \<< EOF > "${HOME}/.cos.yaml"
-            cos:
-              base:
-                secretid: ${TENCENT_COS_SECRETID}
-                secretkey: ${TENCENT_COS_SECRETKEY}
-                protocol: https
-            EOF
+            sudo apt-get update
+            sudo apt install -y gpg ca-certificates
+            RPM_GPG_PRIV_KEY=$(echo $RPM_GPG_PRIV_KEY_BASE64 | base64 -d)
+            echo "${RPM_GPG_PRIV_KEY}" >> /tmp/deb-gpg-publish.private
+            echo "${RPM_GPG_PASSPHRASE}" >> /tmp/deb-gpg-publish.passphrase
+            ./utils/publish-deb.sh init_cos_utils
+            sudo -E ./utils/publish-deb.sh init_freight_utils
+            sudo -E ./utils/publish-deb.sh init_gpg
 
       - when:
           condition:
@@ -152,14 +174,7 @@ jobs:
                   make package type=deb app=apisix openresty=apisix-base checkout=${APISIX_TAG_VERSION} version=${APISIX_TAG_VERSION} image_base=${VAR_OS} image_tag=${VAR_OS_RELEASE}
                   mv ./output/apisix_${APISIX_TAG_VERSION}-0~${VAR_OS}${VAR_OS_RELEASE}_arm64.deb ${VAR_DEB_WORKBENCH_DIR}
 
-            - run:
-                name: upload deb package
-                command: |
-                  find "${VAR_DEB_WORKBENCH_DIR}" -type f -name "apisix_*.deb" \
-                    -exec echo "upload : {}" \; \
-                    -exec sh -c 'file=$(basename {}); \
-                            ./coscli -e "${VAR_COS_ENDPOINT}" \
-                                            cp --part-size 1000 {} "cos://${VAR_COS_BUCKET_REPO}/packages/arm64/debian/pool/main/a/apisix/${file}"' \;
+            - publish_package
 
       - when:
           condition:
@@ -172,14 +187,8 @@ jobs:
                   echo "apisix-base version: $APISIX_BASE_TAG_VERSION"
                   make package type=deb app=apisix-base checkout=${APISIX_BASE_TAG_VERSION} version=${APISIX_BASE_TAG_VERSION} image_base=${VAR_OS} image_tag=${VAR_OS_RELEASE}
                   mv ./output/apisix-base_${APISIX_BASE_TAG_VERSION}-0~${VAR_OS}${VAR_OS_RELEASE}_arm64.deb ${VAR_DEB_WORKBENCH_DIR}
-            - run:
-                name: upload deb package
-                command: |
-                  find "${VAR_DEB_WORKBENCH_DIR}" -type f -name "apisix-base*.deb" \
-                      -exec echo "upload : {}" \; \
-                      -exec sh -c 'file=$(basename {}); \
-                            ./coscli -e "${VAR_COS_ENDPOINT}" \
-                                            cp --part-size 1000 {} "cos://${VAR_COS_BUCKET_REPO}/packages/arm64/debian/pool/main/a/apisix-base/${file}"' \;
+
+            - publish_package
 
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
diff --git a/.github/workflows/publish-deb.yml b/.github/workflows/publish-deb.yml
@@ -13,6 +13,9 @@ jobs:
       VAR_COS_BUCKET_REPO: ${{ secrets.VAR_COS_BUCKET_REPO }}
       VAR_COS_ENDPOINT: ${{ secrets.VAR_COS_ENDPOINT }}
       VAR_DEB_WORKBENCH_DIR: /tmp/output
+      VAR_OS: debian
+      VAR_CODENAME: bullseye
+      VAR_OS_RELEASE: bullseye-slim
 
     steps:
       - name: Check out code
@@ -22,10 +25,9 @@ jobs:
 
       - name: Init basic publish env
         run: |
+          sudo apt-get update
+          sudo apt install -y gpg ca-certificates
           mkdir -p "${VAR_DEB_WORKBENCH_DIR}"
-          # init env var
-          TAG_DATE=$(date +%Y%m%d)
-          echo "TAG_DATE=${TAG_DATE}" >> "$GITHUB_ENV"
 
       - name: Extract Tags name
         if: ${{ startsWith(github.ref, 'refs/tags/') }}
@@ -47,9 +49,8 @@ jobs:
           APISIX_TAG_VERSION: ${{ steps.tag_env.outputs.version}}
         run: |
           # build apisix
-          make package type=deb app=apisix openresty=apisix-base checkout=${APISIX_TAG_VERSION} version=${APISIX_TAG_VERSION} image_base=debian image_tag=bullseye-slim
-          mv ./output/apisix_${APISIX_TAG_VERSION}-0~debianbullseye-slim_amd64.deb ${VAR_DEB_WORKBENCH_DIR}
-          echo "TARGET_APP=apisix" >> "$GITHUB_ENV"
+          make package type=deb app=apisix openresty=apisix-base checkout=${APISIX_TAG_VERSION} version=${APISIX_TAG_VERSION} image_base=${VAR_OS} image_tag=${VAR_OS_RELEASE}
+          mv ./output/apisix_${APISIX_TAG_VERSION}-0~${VAR_OS}${VAR_OS_RELEASE}_amd64.deb ${VAR_DEB_WORKBENCH_DIR}
 
       - name: Build apisix-base deb Package
         if: ${{ startsWith(steps.tag_type.outputs.version, 'apisix-base/') }}
@@ -59,31 +60,52 @@ jobs:
           # build apisix-base
           echo ${{ steps.tag_env.outputs.version}}
           echo ${{ steps.tag_type.outputs.version}}
-          make package type=deb app=apisix-base checkout=${APISIX_BASE_TAG_VERSION} version=${APISIX_BASE_TAG_VERSION} image_base=debian image_tag=bullseye-slim
-          mv ./output/apisix-base_${APISIX_BASE_TAG_VERSION}-0~debianbullseye-slim_amd64.deb ${VAR_DEB_WORKBENCH_DIR}
-          echo "TARGET_APP=apisix-base" >> "$GITHUB_ENV"
+          make package type=deb app=apisix-base checkout=${APISIX_BASE_TAG_VERSION} version=${APISIX_BASE_TAG_VERSION} image_base=${VAR_OS} image_tag=${VAR_OS_RELEASE}
+          mv ./output/apisix-base_${APISIX_BASE_TAG_VERSION}-0~${VAR_OS}${VAR_OS_RELEASE}_amd64.deb ${VAR_DEB_WORKBENCH_DIR}
 
       - name: Upload apisix Artifact
         if: ${{ startsWith(steps.tag_type.outputs.version, 'apisix/') }}
         uses: actions/upload-artifact@v2.2.3
         with:
-          name: "apisix_${{ steps.tag_env.outputs.version}}-0~debianbullseye-slim_amd64.deb"
-          path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix_${{ steps.tag_env.outputs.version}}-0~debianbullseye-slim_amd64.deb"
+          name: "apisix_${{ steps.tag_env.outputs.version}}-0~${{ VAR_OS }}${{ VAR_OS_RELEASE }}_amd64.deb"
+          path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
 
       - name: Upload apisix-base Artifact
         if: ${{ startsWith(steps.tag_type.outputs.version, 'apisix-base/') }}
         uses: actions/upload-artifact@v2.2.3
         with:
-          name: "apisix-base_${{ steps.tag_env.outputs.version}}-0~debianbullseye-slim_amd64.deb"
-          path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix-base_${{ steps.tag_env.outputs.version}}-0~debianbullseye-slim_amd64.deb"
+          name: "apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ VAR_OS }}${{ VAR_OS_RELEASE }}_amd64.deb"
+          path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
 
-      - name: deb repo deps init
+      - name: Init publish utils
         env:
+          GPG_MAIL: ${{ secrets.GPG_MAIL }}
           TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }}
           TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }}
         run: |
-          sudo -E ./utils/publish-deb.sh init_cos_utils
+          echo "${{ secrets.RPM_GPG_PRIV_KEY }}" >> /tmp/deb-gpg-publish.private
+          echo "${{ secrets.RPM_GPG_PASSPHRASE }}" >> /tmp/deb-gpg-publish.passphrase
+          ./utils/publish-deb.sh init_cos_utils
+          sudo -E ./utils/publish-deb.sh init_freight_utils
+          sudo -E ./utils/publish-deb.sh init_gpg
 
-      - name: deb upload
-        run: |
+      - name: backup dists to ci bucket
+        run:
+          sudo -E ./utils/publish-deb.sh dists_backup
+
+      - name: download pool dir to local
+        run:
+          sudo -E ./utils/publish-deb.sh repo_clone
+
+      - name: rebuild deb repo
+        run:
+          sudo -E ./utils/publish-deb.sh repo_rebuild
+
+      - name: upload repo
+        run:
           sudo -E ./utils/publish-deb.sh repo_upload
+
+      - name: pulish repo
+        run:
+          sudo -E ./utils/publish-deb.sh repo_publish
+          sudo -E ./utils/publish-deb.sh repo_backup_remove
diff --git a/utils/publish-deb.sh b/utils/publish-deb.sh
