1
1
name : Manual Benchmark to compare versions
2
+ description : |
3
+ This workflow is used to compare two versions of qdrant using the same dataset and engine config.
4
+ It is triggered manually and requires the user to provide the versions of qdrant to compare, dataset and engine config.
5
+ The workflow will prepare the images for the provided versions (if needed), run the benchmark for each version and compare the results.
2
6
3
7
on :
4
8
workflow_dispatch :
20
24
prepareImage1 :
21
25
name : Prepare image ${{ inputs.qdrant_version_1 }}
22
26
runs-on : ubuntu-latest
27
+ timeout-minutes : 180
23
28
steps :
24
29
- uses : actions/checkout@v3
25
30
-
uses :
webfactory/[email protected]
@@ -29,12 +34,13 @@ jobs:
29
34
run : |
30
35
# The branch, tag or SHA to checkout.
31
36
export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
32
- export BEARER_TOKEN="${{ secrets.GITHUB_TOKEN }}"
37
+ export BEARER_TOKEN="${{ secrets.TRIGGER_GH_TOKEN }}"
33
38
bash -x tools/compare_versions/prepare_image.sh
34
39
35
40
prepareImage2 :
36
- name : Prepare image ${{ inputs.qdrant_version_1 }}
41
+ name : Prepare image ${{ inputs.qdrant_version_2 }}
37
42
runs-on : ubuntu-latest
43
+ timeout-minutes : 180
38
44
steps :
39
45
- uses : actions/checkout@v3
40
46
-
uses :
webfactory/[email protected]
@@ -43,50 +49,126 @@ jobs:
43
49
- name : Image for ${{ inputs.qdrant_version_2 }}
44
50
run : |
45
51
export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
46
- export BEARER_TOKEN="${{ secrets.GITHUB_TOKEN }}"
52
+ export BEARER_TOKEN="${{ secrets.TRIGGER_GH_TOKEN }}"
47
53
bash -x tools/compare_versions/prepare_image.sh
48
54
49
- # runBenchmarkForVersion1:
50
- # name: compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
51
- # needs:
52
- # - prepareImage1
53
- # - prepareImage2
54
- # runs-on: ubuntu-latest
55
- # steps:
56
- # - uses: actions/checkout@v3
57
- # - uses: webfactory/[email protected]
58
- # with:
59
- # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
60
- # - name: Bench ${{ inputs.qdrant_version_1 }}
61
- # run: |
62
- # export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
63
- # export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
64
- # export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
65
- # export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
66
- # export DATASETS=${{ inputs.dataset }}
67
- # export ENGINE_NAME=${{ inputs.engine_config }}
68
- # export POSTGRES_TABLE=benchmark_manual
69
- # bash -x tools/setup_ci.sh
70
- # bash -x tools/run_ci.sh
71
- #
72
- # runBenchmarkForVersion2:
73
- # name: compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
74
- # needs:
75
- # - runBenchmarkForVersion1
76
- # runs-on: ubuntu-latest
77
- # steps:
78
- # - uses: actions/checkout@v3
79
- # - uses: webfactory/[email protected]
80
- # with:
81
- # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
82
- # - name: Bench ${{ inputs.qdrant_version_2 }}
83
- # run: |
84
- # export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
85
- # export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
86
- # export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
87
- # export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
88
- # export DATASETS=${{ inputs.dataset }}
89
- # export ENGINE_NAME=${{ inputs.engine_config }}
90
- # export POSTGRES_TABLE=benchmark_manual
91
- # bash -x tools/setup_ci.sh
92
- # bash -x tools/run_ci.sh
55
+ runBenchmarkForVersion1 :
56
+ name : execute - ${{ inputs.qdrant_version_1 }} - ${{ inputs.dataset }}
57
+ needs :
58
+ - prepareImage1
59
+ runs-on : ubuntu-latest
60
+ concurrency :
61
+ group : continuous-benchmark
62
+ outputs :
63
+ collection_load_time : ${{ steps.bench.outputs.collection_load_time }}
64
+ rps : ${{ steps.bench.outputs.rps }}
65
+ mean_precisions : ${{ steps.bench.outputs.mean_precisions }}
66
+ p95_time : ${{ steps.bench.outputs.p95_time }}
67
+ p99_time : ${{ steps.bench.outputs.p99_time }}
68
+ vm_rss_memory_usage : ${{ steps.bench.outputs.vm_rss_memory_usage }}
69
+ rss_anon_memory_usage : ${{ steps.bench.outputs.rss_anon_memory_usage }}
70
+ upload_time : ${{ steps.bench.outputs.upload_time }}
71
+ indexing_time : ${{ steps.bench.outputs.indexing_time }}
72
+ steps :
73
+ - uses : actions/checkout@v3
74
+ -
uses :
webfactory/[email protected]
75
+ with :
76
+ ssh-private-key : ${{ secrets.SSH_PRIVATE_KEY }}
77
+ - name : Bench ${{ inputs.qdrant_version_1 }}
78
+ id : bench
79
+ run : |
80
+ export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
81
+ export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
82
+ export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
83
+ export QDRANT_VERSION=${{ inputs.qdrant_version_1 }}
84
+ export DATASETS=${{ inputs.dataset }}
85
+ export ENGINE_NAME=${{ inputs.engine_config }}
86
+ export POSTGRES_TABLE=benchmark_manual
87
+ bash -x tools/setup_ci.sh
88
+ bash -x tools/run_ci.sh
89
+
90
+ runBenchmarkForVersion2 :
91
+ name : execute - ${{ inputs.qdrant_version_2 }} - ${{ inputs.dataset }}
92
+ needs :
93
+ - prepareImage2
94
+ - runBenchmarkForVersion1
95
+ runs-on : ubuntu-latest
96
+ concurrency :
97
+ group : continuous-benchmark
98
+ outputs :
99
+ collection_load_time : ${{ steps.bench.outputs.collection_load_time }}
100
+ rps : ${{ steps.bench.outputs.rps }}
101
+ mean_precisions : ${{ steps.bench.outputs.mean_precisions }}
102
+ p95_time : ${{ steps.bench.outputs.p95_time }}
103
+ p99_time : ${{ steps.bench.outputs.p99_time }}
104
+ vm_rss_memory_usage : ${{ steps.bench.outputs.vm_rss_memory_usage }}
105
+ rss_anon_memory_usage : ${{ steps.bench.outputs.rss_anon_memory_usage }}
106
+ upload_time : ${{ steps.bench.outputs.upload_time }}
107
+ indexing_time : ${{ steps.bench.outputs.indexing_time }}
108
+ steps :
109
+ - uses : actions/checkout@v3
110
+ -
uses :
webfactory/[email protected]
111
+ with :
112
+ ssh-private-key : ${{ secrets.SSH_PRIVATE_KEY }}
113
+ - name : Bench ${{ inputs.qdrant_version_2 }}
114
+ id : bench
115
+ run : |
116
+ export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
117
+ export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
118
+ export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
119
+ export QDRANT_VERSION=${{ inputs.qdrant_version_2 }}
120
+ export DATASETS=${{ inputs.dataset }}
121
+ export ENGINE_NAME=${{ inputs.engine_config }}
122
+ export POSTGRES_TABLE=benchmark_manual
123
+ bash -x tools/setup_ci.sh
124
+ bash -x tools/run_ci.sh
125
+
126
+ compareVersions :
127
+ name : compare - ${{ inputs.qdrant_version_1 }} vs ${{ inputs.qdrant_version_2 }}
128
+ needs :
129
+ - runBenchmarkForVersion1
130
+ - runBenchmarkForVersion2
131
+ runs-on : ubuntu-latest
132
+ steps :
133
+ - name : compare
134
+ run : |
135
+ compare() {
136
+ local var_name=$1
137
+ local value_v1=$2
138
+ local value_v2=$3
139
+
140
+ if (( $(echo "$value_v1 > $value_v2" | bc -l) )); then
141
+ local diff=$(echo "$value_v1 - $value_v2" | bc -l)
142
+ local percentage=$(echo "($diff / $value_v1) * 100" | bc -l)
143
+ echo -e "${{ inputs.qdrant_version_1 }} > ${{ inputs.qdrant_version_2 }} by $diff ($percentage% greater)"
144
+ elif (( $(echo "$value_v1 < $value_v2" | bc -l) )); then
145
+ local diff=$(echo "$value_v2 - $value_v1" | bc -l)
146
+ local percentage=$(echo "($diff / $value_v2) * 100" | bc -l)
147
+ echo -e "${{ inputs.qdrant_version_1 }} < ${{ inputs.qdrant_version_2 }} by $diff ($percentage% less)"
148
+ else
149
+ echo -e "equal"
150
+ fi
151
+ }
152
+
153
+ res_collection_load_time=$(compare "collection_load_time" "${{ needs.runBenchmarkForVersion1.outputs.collection_load_time }}" "${{ needs.runBenchmarkForVersion2.outputs.collection_load_time }}")
154
+ res_rps=$(compare "rps" "${{ needs.runBenchmarkForVersion1.outputs.rps }}" "${{ needs.runBenchmarkForVersion2.outputs.rps }}")
155
+ res_mean_precisions=$(compare "mean_precisions" "${{ needs.runBenchmarkForVersion1.outputs.mean_precisions }}" "${{ needs.runBenchmarkForVersion2.outputs.mean_precisions }}")
156
+ res_p95_time=$(compare "p95_time" "${{ needs.runBenchmarkForVersion1.outputs.p95_time }}" "${{ needs.runBenchmarkForVersion2.outputs.p95_time }}")
157
+ res_p99_time=$(compare "p99_time" "${{ needs.runBenchmarkForVersion1.outputs.p99_time }}" "${{ needs.runBenchmarkForVersion2.outputs.p99_time }}")
158
+ res_vm_rss_memory_usage=$(compare "vm_rss_memory_usage" "${{ needs.runBenchmarkForVersion1.outputs.vm_rss_memory_usage }}" "${{ needs.runBenchmarkForVersion2.outputs.vm_rss_memory_usage }}")
159
+ res_rss_anon_memory_usage=$(compare "rss_anon_memory_usage" "${{ needs.runBenchmarkForVersion1.outputs.rss_anon_memory_usage }}" "${{ needs.runBenchmarkForVersion2.outputs.rss_anon_memory_usage }}")
160
+ res_upload_time=$(compare "upload_time" "${{ needs.runBenchmarkForVersion1.outputs.upload_time }}" "${{ needs.runBenchmarkForVersion2.outputs.upload_time }}")
161
+ res_indexing_time=$(compare "indexing_time" "${{ needs.runBenchmarkForVersion1.outputs.indexing_time }}" "${{ needs.runBenchmarkForVersion2.outputs.indexing_time }}")
162
+
163
+ echo "# Comparison results" >> $GITHUB_STEP_SUMMARY
164
+ echo "| Name | ${{ inputs.qdrant_version_1 }} | ${{ inputs.qdrant_version_2 }} | Result |" >> $GITHUB_STEP_SUMMARY
165
+ echo "| --------------------- | ------------ | ------------ | ------------ |" >> $GITHUB_STEP_SUMMARY
166
+ echo "| collection_load_time | ${{ needs.runBenchmarkForVersion1.outputs.collection_load_time }} | ${{ needs.runBenchmarkForVersion2.outputs.collection_load_time }} | ${res_collection_load_time} |" >> $GITHUB_STEP_SUMMARY
167
+ echo "| rps | ${{ needs.runBenchmarkForVersion1.outputs.rps }} | ${{ needs.runBenchmarkForVersion2.outputs.rps }} | ${res_rps} |" >> $GITHUB_STEP_SUMMARY
168
+ echo "| mean_precisions | ${{ needs.runBenchmarkForVersion1.outputs.mean_precisions }} | ${{ needs.runBenchmarkForVersion2.outputs.mean_precisions }} | ${res_mean_precisions} |" >> $GITHUB_STEP_SUMMARY
169
+ echo "| p95_time | ${{ needs.runBenchmarkForVersion1.outputs.p95_time }} | ${{ needs.runBenchmarkForVersion2.outputs.p95_time }} | ${res_p95_time} |" >> $GITHUB_STEP_SUMMARY
170
+ echo "| p99_time | ${{ needs.runBenchmarkForVersion1.outputs.p99_time }} | ${{ needs.runBenchmarkForVersion2.outputs.p99_time }} | ${res_p99_time} |" >> $GITHUB_STEP_SUMMARY
171
+ echo "| vm_rss_memory_usage | ${{ needs.runBenchmarkForVersion1.outputs.vm_rss_memory_usage }} | ${{ needs.runBenchmarkForVersion2.outputs.vm_rss_memory_usage }} | ${res_vm_rss_memory_usage} |" >> $GITHUB_STEP_SUMMARY
172
+ echo "| rss_anon_memory_usage | ${{ needs.runBenchmarkForVersion1.outputs.rss_anon_memory_usage }} | ${{ needs.runBenchmarkForVersion2.outputs.rss_anon_memory_usage }} | ${res_rss_anon_memory_usage} |" >> $GITHUB_STEP_SUMMARY
173
+ echo "| upload_time | ${{ needs.runBenchmarkForVersion1.outputs.upload_time }} | ${{ needs.runBenchmarkForVersion2.outputs.upload_time }} | ${res_upload_time} |" >> $GITHUB_STEP_SUMMARY
174
+ echo "| indexing_time | ${{ needs.runBenchmarkForVersion1.outputs.indexing_time }} | ${{ needs.runBenchmarkForVersion2.outputs.indexing_time }} | ${res_indexing_time} |" >> $GITHUB_STEP_SUMMARY
0 commit comments