Merge pull request #10518 from everpcpc/feat-bench

everpcpc · web-flow · commit d11512a3d995 · 2023-03-13T13:58:01.000+08:00
chore(ci): fix download artifacts for benchmark
diff --git a/.github/actions/benchmark_local/action.yml b/.github/actions/benchmark_local/action.yml
@@ -19,21 +19,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
-      uses: ./.github/actions/artifact_download
-      with:
-        profile: release
-        sha: ${{ inputs.sha }}
-        target: x86_64-unknown-linux-gnu
-
-    - name: Setup Databend Binary
-      shell: bash
-      run: |
-        sudo cp ./target/release/databend-query /usr/local/bin/databend-query
-        sudo cp ./target/release/databend-meta /usr/local/bin/databend-meta
-        databend-query --version
-        databend-meta --version
-
     - uses: ./.github/actions/setup_bendsql
 
     - name: Install script dependencies
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -15,12 +15,6 @@ on:
       - "**.md"
       - "docker/**"
       - ".devcontainer/**"
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: The tag to run benchmark
-        required: true
-        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -29,13 +23,10 @@ concurrency:
 env:
   BUILD_PROFILE: release
   RUNNER_PROVIDER: aws
-  CARGO_INCREMENTAL: 0
-  # Remove this env after rust 1.68 has been released.
-  CARGO_UNSTABLE_SPARSE_REGISTRY: true
 
 jobs:
   build_release:
-    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-benchmark')
+    if: contains(github.event.pull_request.labels.*.name, 'ci-benchmark')
     timeout-minutes: 30
     runs-on: [self-hosted, X64, Linux, perf]
     steps:
@@ -47,29 +38,12 @@ jobs:
         with:
           target: x86_64-unknown-linux-gnu
           profile: release
-      - name: Get benchmark source
-        run: |
-          case ${{ github.event_name }} in
-            pull_request)
-              echo "BENCHMARK_SOURCE=pr" > $GITHUB_ENV
-              echo "BENCHMARK_SOURCE_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-              ;;
-            workflow_dispatch)
-              echo "BENCHMARK_SOURCE=release" > $GITHUB_ENV
-              echo "BENCHMARK_SOURCE_ID=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
-              ;;
-            *)
-              echo "Unknown event name: ${{ github.event_name }}"
-              exit 1
-              ;;
-          esac
       - name: Generate benchmark metadata
         run: |
           echo '{}' > metadata.json
           jq ".sha = \"${{ github.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
           jq ".run_id = \"${{ github.run_id }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
-          jq ".source = \"${{ env.BENCHMARK_SOURCE }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
-          jq ".source_id = \"${{ env.BENCHMARK_SOURCE_ID }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+          jq ".pr_id = \"${{ github.event.pull_request.number }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -76,14 +76,6 @@ jobs:
           gh release create ${{ steps.generated-tag.outputs.tag }} --generate-notes -p
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Trigger benchmark
-        continue-on-error: true
-        run: |
-          echo "Trigger benchmark for ${{ steps.generated-tag.outputs.tag }}"
-          gh workflow run benchmark.yml -f tag=${{ steps.generated-tag.outputs.tag }}
-        env:
-          GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }}
-
 
   publish_macos:
     name: macos assets
@@ -521,6 +513,14 @@ jobs:
           asset_name: sha256sums.txt
           asset_content_type: text/plain
 
+  run_benchmark:
+    name: run benchmark
+    needs: [create_release, publish_linux]
+    uses: ./.github/workflows/trusted-benchmark.yml
+    secrets: inherit
+    with:
+      tag: ${{ needs.create_release.outputs.version }}
+
   notify_build_result:
     name: notify build result
     runs-on: ubuntu-latest
diff --git a/.github/workflows/trusted-benchmark.yml b/.github/workflows/trusted-benchmark.yml
@@ -5,6 +5,12 @@ on:
     workflows: ["Benchmark"]
     types:
       - completed
+  workflow_call:
+    inputs:
+      tag:
+        description: The tag to run benchmark
+        required: true
+        type: string
 
 permissions:
   id-token: write
@@ -30,11 +36,26 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh run download ${{ github.event.workflow_run.id }} --name benchmark-metadata
-          echo "sha=$(jq -r '.sha' metadata.json)" >> $GITHUB_OUTPUT
-          echo "run_id=$(jq -r '.run_id' metadata.json)" >> $GITHUB_OUTPUT
-          echo "source=$(jq -r '.source' metadata.json)" >> $GITHUB_OUTPUT
-          echo "source_id=$(jq -r '.source_id' metadata.json)" >> $GITHUB_OUTPUT
+          case ${{ github.event_name }} in
+            # pull request
+            workflow_run)
+              gh run download ${{ github.event.workflow_run.id }} --name benchmark-metadata
+              echo "sha=$(jq -r '.sha' metadata.json)" >> $GITHUB_OUTPUT
+              echo "run_id=$(jq -r '.run_id' metadata.json)" >> $GITHUB_OUTPUT
+              echo "source=pr" >> $GITHUB_OUTPUT
+              echo "source_id=$(jq -r '.pr_id' metadata.json)" >> $GITHUB_OUTPUT
+              ;;
+            workflow_call)
+              echo "sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+              echo "run_id=${{ github.run_id }}" >> $GITHUB_OUTPUT
+              echo "source=release" >> $GITHUB_OUTPUT
+              echo "source_id=${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT
+              ;;
+            *)
+              echo "Unknown event name: ${{ github.event_name }}"
+              exit 1
+              ;;
+          esac
 
   local:
     needs: metadata
@@ -47,6 +68,32 @@ jobs:
           - tpch
     steps:
       - uses: actions/checkout@v3
+      - name: Download artifact
+        run: |
+          mkdir -p ./target/release
+          case ${{ needs.metadata.outputs.source }} in
+            pr)
+              aws s3 sync s3://databend-ci/release/${{ needs.metadata.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
+                target/release/ --exclude "*" --include "databend-*" --no-progress
+              ;;
+            release)
+              version=${{ needs.metadata.outputs.source_id }}
+              target=x86_64-unknown-linux-gnu
+              wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz
+              tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
+              ;;
+            *)
+              echo "Unknown source: ${{ needs.metadata.outputs.source }}"
+              exit 1
+              ;;
+          esac
+          chmod +x ./target/release/databend-*
+      - name: Setup Databend Binary
+        shell: bash
+        run: |
+          sudo cp ./target/release/databend-* /usr/local/bin/
+          databend-query --version
+          databend-meta --version
       - uses: ./.github/actions/benchmark_local
         with:
           sha: ${{ needs.metadata.outputs.sha }}
@@ -63,17 +110,35 @@ jobs:
       tag: ${{ steps.prepare.outputs.tag }}
     steps:
       - uses: actions/checkout@v3
+      - name: Download artifact
+        run: |
+          mkdir -p ./target/release
+          case ${{ needs.metadata.outputs.source }} in
+            pr)
+              aws s3 sync s3://databend-ci/release/${{ needs.metadata.outputs.sha }}/x86_64-unknown-linux-gnu/default/ \
+                target/release/ --exclude "*" --include "databend-*" --no-progress
+              ;;
+            release)
+              version=${{ needs.metadata.outputs.source_id }}
+              target=x86_64-unknown-linux-gnu
+              wget -P distro -qc https://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz
+              tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
+              ;;
+            *)
+              echo "Unknown source: ${{ needs.metadata.outputs.source }}"
+              exit 1
+              ;;
+          esac
+          chmod +x ./target/release/databend-*
+      - name: Prepare docker build
+        run: |
+          mkdir -p ./distro/linux/amd64
+          cp ./target/release/databend-* ./distro/linux/amd64
       - uses: ./.github/actions/setup_docker
         id: login
         with:
           repo: databend-query
           ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
-      - name: Download artifact
-        uses: ./.github/actions/artifact_download
-        with:
-          profile: release
-          sha: ${{ needs.metadata.outputs.sha }}
-          target: x86_64-unknown-linux-gnu
       - name: Prepare for docker
         id: prepare
         run: |
@@ -91,7 +156,7 @@ jobs:
 
   cloud:
     needs: [metadata, docker]
-    timeout-minutes: 30
+    timeout-minutes: 60
     runs-on: [self-hosted, X64, Linux, dev]
     steps:
       - uses: actions/checkout@v3
diff --git a/benchmark/clickbench/benchmark_cloud.sh b/benchmark/clickbench/benchmark_cloud.sh
@@ -39,7 +39,7 @@ bendsql cloud login \
     --org "${CLOUD_ORG}"
 
 bendsql cloud warehouse ls
-bendsql cloud warehouse create "${CLOUD_WAREHOUSE}" --size 1 --tag "${BENCHMARK_IMAGE_TAG}"
+bendsql cloud warehouse create "${CLOUD_WAREHOUSE}" --size "${BENCHMARK_SIZE}" --tag "${BENCHMARK_IMAGE_TAG}"
 bendsql cloud warehouse ls
 bendsql cloud warehouse resume "${CLOUD_WAREHOUSE}" --wait
 
@@ -104,4 +104,4 @@ while read -r query; do
 done <"${BENCHMARK_DATASET}/queries.sql"
 
 echo "Cleaning up..."
-bendsql cloud warehouse delete: "${CLOUD_WAREHOUSE}"
+bendsql cloud warehouse delete "${CLOUD_WAREHOUSE}"
